From ce2987fe6209f8aaf1cbe76363b462c2ba49b7b8 Mon Sep 17 00:00:00 2001 From: Fisher Darling Date: Tue, 9 Nov 2021 09:08:18 +0000 Subject: [PATCH] add a NodeId to ast components Signed-off-by: Fisher Darling --- tremor-pipeline/src/op/trickle/operator.rs | 2 +- tremor-pipeline/src/query.rs | 38 ++++---- tremor-script/src/ast.rs | 1 + tremor-script/src/ast/node_id.rs | 102 +++++++++++++++++++++ tremor-script/src/ast/query.rs | 98 +++++--------------- tremor-script/src/ast/query/raw.rs | 45 ++++----- tremor-script/src/ast/to_static.rs | 12 +-- 7 files changed, 170 insertions(+), 128 deletions(-) create mode 100644 tremor-script/src/ast/node_id.rs diff --git a/tremor-pipeline/src/op/trickle/operator.rs b/tremor-pipeline/src/op/trickle/operator.rs index 0954c7b3f4..67ff25f268 100644 --- a/tremor-pipeline/src/op/trickle/operator.rs +++ b/tremor-pipeline/src/op/trickle/operator.rs @@ -55,7 +55,7 @@ impl TrickleOperator { ast::Stmt::OperatorDecl(ref op) => { let op = op.clone().into_static(); let config = mk_node_config( - op.id, + op.node_id.id().to_string(), format!("{}::{}", op.kind.module, op.kind.operation), op.params, ); diff --git a/tremor-pipeline/src/query.rs b/tremor-pipeline/src/query.rs index 58d5f73058..d9f0b41961 100644 --- a/tremor-pipeline/src/query.rs +++ b/tremor-pipeline/src/query.rs @@ -426,24 +426,21 @@ impl Query { subqueries.insert(s.id.clone(), s.clone()); } Stmt::Operator(o) => { - if nodes.contains_key(&common_cow(&o.id)) { - let error_func = if has_builtin_node_name(&common_cow(&o.id)) { + if nodes.contains_key(&common_cow(o.node_id.id())) { + let error_func = if has_builtin_node_name(&common_cow(o.node_id.id())) { query_node_reserved_name_err } else { query_node_duplicate_name_err }; - return Err(error_func(o, o.id.clone(), &query.node_meta).into()); + return Err( + error_func(o, o.node_id.id().to_string(), &query.node_meta).into() + ); } - let target = o.target.clone().to_string(); - let fqon = if o.module.is_empty() { - target - } else { - format!("{}::{}", o.module.join("::"), target) - }; + let fqon = o.node_id.target_fqn(&o.target); let node = NodeConfig { - id: o.id.clone(), + id: o.node_id.id().to_string(), kind: NodeKind::Operator, op_type: "trickle::operator".to_string(), ..NodeConfig::default() @@ -482,25 +479,22 @@ impl Query { None, )?; pipe_ops.insert(id, op); - nodes.insert(common_cow(&o.id), id); + nodes.insert(common_cow(o.node_id.id()), id); outputs.push(id); } Stmt::Script(o) => { - if nodes.contains_key(&common_cow(&o.id)) { - let error_func = if has_builtin_node_name(&common_cow(&o.id)) { + if nodes.contains_key(&common_cow(o.node_id.id())) { + let error_func = if has_builtin_node_name(&common_cow(o.node_id.id())) { query_node_reserved_name_err } else { query_node_duplicate_name_err }; - return Err(error_func(o, o.id.clone(), &query.node_meta).into()); + return Err( + error_func(o, o.node_id.id().to_string(), &query.node_meta).into() + ); } - let target = o.target.clone().to_string(); - let fqsn = if o.module.is_empty() { - target - } else { - format!("{}::{}", o.module.join("::"), target) - }; + let fqsn = o.node_id.target_fqn(&o.target); let stmt_srs = srs::Stmt::try_new_from_query::<&'static str, _>(&self.0.query, |query| { @@ -524,7 +518,7 @@ impl Query { let that_defn = stmt_srs; let node = NodeConfig { - id: o.id.clone(), + id: o.node_id.id().to_string(), kind: NodeKind::Script, label, op_type: "trickle::script".to_string(), @@ -542,7 +536,7 @@ impl Query { None, )?; pipe_ops.insert(id, op); - nodes.insert(common_cow(&o.id), id); + nodes.insert(common_cow(o.node_id.id()), id); outputs.push(id); } }; diff --git a/tremor-script/src/ast.rs b/tremor-script/src/ast.rs index ecfb7d9b8e..bba9d67386 100644 --- a/tremor-script/src/ast.rs +++ b/tremor-script/src/ast.rs @@ -19,6 +19,7 @@ pub(crate) mod binary; /// custom equality definition - checking for equivalence of different AST nodes /// e.g. two different event paths with different metadata pub mod eq; +mod node_id; /// Query AST pub mod query; pub(crate) mod raw; diff --git a/tremor-script/src/ast/node_id.rs b/tremor-script/src/ast/node_id.rs new file mode 100644 index 0000000000..00f2febf3f --- /dev/null +++ b/tremor-script/src/ast/node_id.rs @@ -0,0 +1,102 @@ +// Copyright 2020-2021, The Tremor Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/// Identifies a node in the AST. +#[derive(Clone, Debug, PartialEq, Serialize)] +pub struct NodeId { + /// The ID of the Node + id: String, + /// The module of the Node + module: Vec, +} + +impl NodeId { + /// Create a new `NodeId` from an ID and Module list. + pub fn new(id: String, module: Vec) -> Self { + Self { id, module } + } + + /// The node's id. + pub fn id(&self) -> &str { + self.id.as_str() + } + + /// The node's module. + pub fn module(&self) -> &[String] { + &self.module + } + + /// Mutate the node's module. + pub fn module_mut(&mut self) -> &mut Vec { + &mut self.module + } + + /// Calculate the fully qualified name from + /// the given module path. + #[must_use] + pub fn fqn(&self) -> String { + if self.module.is_empty() { + self.id.to_string() + } else { + format!("{}::{}", self.module.join("::"), self.id) + } + } + + /// Calculate the fully qualified name of some + /// target identifier given this node's module + /// path. + #[must_use] + pub fn target_fqn(&self, target: &str) -> String { + if self.module.is_empty() { + target.to_string() + } else { + format!("{}::{}", self.module.join("::"), target) + } + } +} + +#[doc(hidden)] +#[macro_export] +macro_rules! impl_fqn { + ($struct:ident) => { + impl $struct<'_> { + fn fqn(&self) -> String { + self.node_id.fqn() + } + } + }; +} + +#[cfg(test)] +mod test { + use super::NodeId; + + #[test] + fn fqn() { + let no_module = NodeId::new("foo".to_string(), vec![]); + assert_eq!(no_module.fqn(), "foo"); + assert!(no_module.module().is_empty()); + + let with_module = NodeId::new( + "foo".to_string(), + vec!["bar".to_string(), "baz".to_string()], + ); + assert_eq!(with_module.fqn(), "bar::baz::foo"); + assert_eq!(with_module.module(), &["bar", "baz"]); + + let target = "quux"; + assert_eq!(no_module.target_fqn(target), target); + assert_eq!(with_module.target_fqn(target), "bar::baz::quux"); + } +} diff --git a/tremor-script/src/ast/query.rs b/tremor-script/src/ast/query.rs index b442c2952e..44c6771097 100644 --- a/tremor-script/src/ast/query.rs +++ b/tremor-script/src/ast/query.rs @@ -14,13 +14,13 @@ pub(crate) mod raw; use super::{ - error_generic, error_no_consts, error_no_locals, AggrRegistry, EventPath, HashMap, Helper, - Ident, ImutExpr, ImutExprInt, InvokeAggrFn, Location, NodeMetas, Path, Registry, Result, - Script, Serialize, Stmts, Upable, Value, + error_generic, error_no_consts, error_no_locals, node_id::NodeId, AggrRegistry, EventPath, + HashMap, Helper, Ident, ImutExpr, ImutExprInt, InvokeAggrFn, Location, NodeMetas, Path, + Registry, Result, Script, Serialize, Stmts, Upable, Value, }; use super::{raw::BaseExpr, Consts}; -use crate::ast::eq::AstEq; use crate::impl_expr_mid; +use crate::{ast::eq::AstEq, impl_fqn}; use raw::WindowDefnRaw; /// A Tremor query @@ -164,40 +164,27 @@ impl BaseExpr for OperatorKind { /// An operator declaration #[derive(Clone, Debug, PartialEq, Serialize)] pub struct OperatorDecl<'script> { + /// The ID and Module of the Operator + pub node_id: NodeId, + /// metadata id pub(crate) mid: usize, /// Type of the operator pub kind: OperatorKind, - /// Module of the operator - pub module: Vec, - /// Identifer for the operator - pub id: String, /// Parameters for the operator pub params: Option>>, } impl_expr_mid!(OperatorDecl); - -impl<'script> OperatorDecl<'script> { - /// Calculate the fully qualified name - #[must_use] - pub fn fqon(&self, module: &[String]) -> String { - if module.is_empty() { - self.id.clone() - } else { - format!("{}::{}", module.join("::"), self.id) - } - } -} +impl_fqn!(OperatorDecl); /// An operator creation #[derive(Clone, Debug, PartialEq, Serialize)] pub struct OperatorStmt<'script> { + /// The ID and Module of the Operator + pub node_id: NodeId, + /// metadata id pub(crate) mid: usize, - /// Id of the operator - pub id: String, /// Target of the operator pub target: String, - /// Module of the script - pub module: Vec, /// parameters of the instance pub params: Option>>, } @@ -207,10 +194,8 @@ impl_expr_mid!(OperatorStmt); #[derive(Clone, Debug, PartialEq, Serialize)] pub struct ScriptDecl<'script> { pub(crate) mid: usize, - /// Module of the script - pub module: Vec, - /// ID of the script - pub id: String, + /// The ID and Module of the Script + pub node_id: NodeId, /// Parameters of a script declaration pub params: Option>>, /// The script itself @@ -218,41 +203,27 @@ pub struct ScriptDecl<'script> { } impl_expr_mid!(ScriptDecl); -impl<'script> ScriptDecl<'script> { - /// Calculate the fully qualified name - #[must_use] - pub fn fqsn(&self, module: &[String]) -> String { - if module.is_empty() { - self.id.clone() - } else { - format!("{}::{}", module.join("::"), self.id) - } - } -} - /// A script creation #[derive(Clone, Debug, PartialEq, Serialize)] pub struct ScriptStmt<'script> { + /// The ID and Module of the Script + pub node_id: NodeId, + /// metadata id pub(crate) mid: usize, - /// ID of the script - pub id: String, /// Target of the script pub target: String, /// Parameters of the script statement pub params: Option>>, - /// Module path of the script - pub module: Vec, } impl_expr_mid!(ScriptStmt); /// A subquery declaration #[derive(Clone, Debug, PartialEq, Serialize)] pub struct SubqueryDecl<'script> { + /// The ID and Module of the SubqueryDecl + pub node_id: NodeId, + /// metadata id pub(crate) mid: usize, - /// Module of the subquery - pub module: Vec, - /// ID of the subquery - pub id: String, /// Parameters of a subquery declaration pub params: Option>>, /// Input Ports @@ -263,18 +234,7 @@ pub struct SubqueryDecl<'script> { pub raw_stmts: raw::StmtsRaw<'script>, } impl_expr_mid!(SubqueryDecl); - -impl<'script> SubqueryDecl<'script> { - /// Calculate the fully qualified name - #[must_use] - pub fn fqsqn(&self, module: &[String]) -> String { - if module.is_empty() { - self.id.clone() - } else { - format!("{}::{}", module.join("::"), self.id) - } - } -} +impl_fqn!(SubqueryDecl); /// A subquery creation #[derive(Clone, Debug, PartialEq, Serialize)] @@ -305,12 +265,10 @@ pub enum WindowKind { /// A window declaration #[derive(Clone, Debug, PartialEq, Serialize)] pub struct WindowDecl<'script> { + /// ID and Module of the Window + pub node_id: NodeId, /// metadata id pub(crate) mid: usize, - /// Module of the window declaration - pub module: Vec, - /// Name of the window declaration - pub id: String, /// The type of window pub kind: WindowKind, /// Parameters passed to the window @@ -319,6 +277,7 @@ pub struct WindowDecl<'script> { pub script: Option>, } impl_expr_mid!(WindowDecl); +impl_fqn!(WindowDecl); impl<'script> WindowDecl<'script> { /// `emit_empty_windows` setting @@ -329,16 +288,6 @@ impl<'script> WindowDecl<'script> { pub const INTERVAL: &'static str = "interval"; /// `size` setting pub const SIZE: &'static str = "size"; - - /// Calculate the fully qualified window name - #[must_use] - pub fn fqwn(&self, module: &[String]) -> String { - if module.is_empty() { - self.id.clone() - } else { - format!("{}::{}", module.join("::"), self.id) - } - } } /// A select statement @@ -354,7 +303,6 @@ pub struct Select<'script> { pub target: ImutExpr<'script>, /// Where clause pub maybe_where: Option>, - /// Having clause pub maybe_having: Option>, /// Group-By clause diff --git a/tremor-script/src/ast/query/raw.rs b/tremor-script/src/ast/query/raw.rs index db0d2f3247..fb7e35d30d 100644 --- a/tremor-script/src/ast/query/raw.rs +++ b/tremor-script/src/ast/query/raw.rs @@ -25,6 +25,7 @@ use super::{ StreamStmt, SubqueryDecl, SubqueryStmt, Upable, Value, WindowDecl, WindowKind, }; use crate::ast::{ + node_id::NodeId, visitors::{ArgsRewriter, ExprReducer, GroupByExprExtractor, TargetEventRef}, Ident, }; @@ -198,15 +199,14 @@ impl<'script> Upable<'script> for OperatorDeclRaw<'script> { fn up<'registry>(self, helper: &mut Helper<'script, 'registry>) -> Result { let operator_decl = OperatorDecl { mid: helper.add_meta_w_name(self.start, self.end, &self.id), - module: helper.module.clone(), - id: self.id, + node_id: NodeId::new(self.id, helper.module.clone()), kind: self.kind.up(helper)?, params: self.params.map(|raw| raw.up(helper)).transpose()?, }; helper .operators - .insert(operator_decl.fqon(&helper.module), operator_decl.clone()); - helper.add_query_decl_doc(&operator_decl.id, self.doc); + .insert(operator_decl.fqn(), operator_decl.clone()); + helper.add_query_decl_doc(&operator_decl.node_id.id(), self.doc); Ok(operator_decl) } } @@ -257,14 +257,13 @@ impl<'script> Upable<'script> for SubqueryDeclRaw<'script> { let subquery_decl = SubqueryDecl { mid: helper.add_meta_w_name(self.start, self.end, &self.id), - module: helper.module.clone(), - id: self.id, + node_id: NodeId::new(self.id, helper.module.clone()), params: self.params.map(|raw| raw.up(helper)).transpose()?, raw_stmts: self.subquery, from, into, }; - let subquery_name = subquery_decl.fqsqn(&helper.module); + let subquery_name = subquery_decl.fqn(); if helper.subquery_defns.contains_key(&subquery_name) { let err_str = format! {"Can't define the query `{}` twice", subquery_name}; return error_generic(&subquery_decl, &subquery_decl, &err_str, &helper.meta); @@ -273,7 +272,7 @@ impl<'script> Upable<'script> for SubqueryDeclRaw<'script> { helper .subquery_defns .insert(subquery_name, subquery_decl.clone()); - helper.add_query_decl_doc(&subquery_decl.id, self.doc); + helper.add_query_decl_doc(&subquery_decl.node_id.id(), self.doc); Ok(subquery_decl) } } @@ -449,7 +448,7 @@ impl<'script> SubqueryStmtRaw<'script> { meta.name = Some(unmangled_id); } // All `define`s inside the subq are inside `subq_module` - o.module.insert(0, subq_module.clone()); + o.node_id.module_mut().insert(0, subq_module.clone()); query_stmts.push(Stmt::Operator(o)); } StmtRaw::Script(mut s) => { @@ -466,7 +465,7 @@ impl<'script> SubqueryStmtRaw<'script> { if let Some(meta) = helper.meta.nodes.get_mut(s.mid) { meta.name = Some(unmangled_id); } - s.module.insert(0, subq_module.clone()); + s.node_id.module_mut().insert(0, subq_module.clone()); query_stmts.push(Stmt::Script(s)); } StmtRaw::Stream(mut s) => { @@ -671,8 +670,7 @@ impl<'script> Upable<'script> for OperatorStmtRaw<'script> { let module = self.module.iter().map(ToString::to_string).collect(); Ok(OperatorStmt { mid: helper.add_meta_w_name(self.start, self.end, &self.id), - id: self.id, - module, + node_id: NodeId::new(self.id, module), target: self.target, params: self.params.map(|raw| raw.up(helper)).transpose()?, }) @@ -706,15 +704,20 @@ impl<'script> Upable<'script> for ScriptDeclRaw<'script> { let script_decl = ScriptDecl { mid: helper.add_meta_w_name(self.start, self.end, &self.id), - module: helper.module.clone(), - id: self.id, + node_id: NodeId::new(self.id, helper.module.clone()), params: self.params.map(|raw| raw.up(helper)).transpose()?, script, }; helper.module.pop(); - let script_name = script_decl.fqsn(&helper.module); + + let script_name = if helper.module.is_empty() { + script_decl.node_id.id().to_string() + } else { + format!("{}::{}", helper.module.join("::"), script_decl.node_id.id()) + }; + helper.scripts.insert(script_name, script_decl.clone()); - helper.add_query_decl_doc(&script_decl.id, self.doc); + helper.add_query_decl_doc(&script_decl.node_id.id(), self.doc); Ok(script_decl) } } @@ -736,10 +739,9 @@ impl<'script> Upable<'script> for ScriptStmtRaw<'script> { let module = self.module.iter().map(ToString::to_string).collect(); Ok(ScriptStmt { mid: helper.add_meta_w_name(self.start, self.end, &self.id), - id: self.id, + node_id: NodeId::new(self.id, module), params: self.params.map(|raw| raw.up(helper)).transpose()?, target: self.target, - module, }) } } @@ -765,8 +767,7 @@ impl<'script> Upable<'script> for WindowDeclRaw<'script> { let window_decl = WindowDecl { mid: helper.add_meta_w_name(self.start, self.end, &self.id), - module: helper.module.clone(), - id: self.id, + node_id: NodeId::new(self.id, helper.module.clone()), kind: self.kind, params: self.params.up(helper)?, script: maybe_script, @@ -774,8 +775,8 @@ impl<'script> Upable<'script> for WindowDeclRaw<'script> { helper .windows - .insert(window_decl.fqwn(&helper.module), window_decl.clone()); - helper.add_query_decl_doc(&window_decl.id, self.doc); + .insert(window_decl.fqn(), window_decl.clone()); + helper.add_query_decl_doc(&window_decl.node_id.id(), self.doc); Ok(window_decl) } } diff --git a/tremor-script/src/ast/to_static.rs b/tremor-script/src/ast/to_static.rs index 2a18594e02..17df12ebbf 100644 --- a/tremor-script/src/ast/to_static.rs +++ b/tremor-script/src/ast/to_static.rs @@ -1052,17 +1052,15 @@ impl<'script> WindowDecl<'script> { #[must_use] pub fn into_static(self) -> WindowDecl<'static> { let WindowDecl { + node_id, mid, - module, - id, kind, params, script, } = self; WindowDecl { + node_id, mid, - module, - id, kind, params: params .into_iter() @@ -1079,17 +1077,15 @@ impl<'script> OperatorDecl<'script> { #[must_use] pub fn into_static(self) -> OperatorDecl<'static> { let OperatorDecl { + node_id, mid, kind, - module, - id, params, } = self; OperatorDecl { + node_id, mid, kind, - module, - id, params: params.map(|params| { params .into_iter()