Skip to content

Implementation of modules/use statements #660

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

Implementation of modules/use statements #660

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions derive/tests/grammar.pest
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,3 +84,5 @@ COMMENT = _{ "$"+ }

WHITESPACE = _{ "hi" }
*/

mod x { y = { "y" } }
6 changes: 6 additions & 0 deletions generator/src/generator.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -473,6 +473,9 @@ fn generate_expr(expr: OptimizedExpr) -> TokenStream {
state.stack_push(|state| #expr)
}
}
OptimizedExpr::Module(name, exprs) => {
quote! {}
}
OptimizedExpr::RestoreOnErr(expr) => {
let expr = generate_expr(*expr);

Expand Down Expand Up @@ -605,6 +608,9 @@ fn generate_expr_atomic(expr: OptimizedExpr) -> TokenStream {
state.stack_push(|state| #expr)
}
}
OptimizedExpr::Module(name, exprs) => {
quote! {}
}
OptimizedExpr::RestoreOnErr(expr) => {
let expr = generate_expr_atomic(*expr);

Expand Down
4 changes: 3 additions & 1 deletion generator/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,8 +91,10 @@ pub fn derive_parser(input: TokenStream, include_grammar: bool) -> TokenStream {
),
};

let defaults = unwrap_or_report(validator::validate_pairs(pairs.clone()));
let defaults = validator::validate_pairs(pairs.clone()).unwrap();

let ast = unwrap_or_report(parser::consume_rules(pairs));
dbg!(&ast);
let optimized = optimizer::optimize(ast);

generator::generate(name, &generics, path, optimized, defaults, include_grammar)
Expand Down
38 changes: 36 additions & 2 deletions meta/src/ast.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@
// option. All files in the project carrying such notice may not be copied,
// modified, or distributed except according to those terms.

use std::vec::IntoIter;

#[derive(Clone, Debug, Eq, PartialEq)]
pub struct Rule {
pub name: String,
Expand Down Expand Up @@ -61,6 +63,8 @@ pub enum Expr {
Skip(Vec<String>),
/// Matches an expression and pushes it to the stack, e.g. `push(e)`
Push(Box<Expr>),
/// A named module that contains numerous `Expr`s inside.
Module(String, Vec<Expr>),
}

impl Expr {
Expand All @@ -79,7 +83,6 @@ impl Expr {
let expr = f(expr);

match expr {
// TODO: Use box syntax when it gets stabilized.
Expr::PosPred(expr) => {
let mapped = Box::new(map_internal(*expr, f));
Expr::PosPred(mapped)
Expand Down Expand Up @@ -130,6 +133,13 @@ impl Expr {
let mapped = Box::new(map_internal(*expr, f));
Expr::Push(mapped)
}
Expr::Module(name, exprs) => {
let mut mapped = vec![];
for expr in exprs {
mapped.push(map_internal(expr, f))
}
Expr::Module(name, mapped)
}
expr => expr,
}
}
Expand All @@ -147,7 +157,6 @@ impl Expr {
{
let mapped = match expr {
Expr::PosPred(expr) => {
// TODO: Use box syntax when it gets stabilized.
let mapped = Box::new(map_internal(*expr, f));
Expr::PosPred(mapped)
}
Expand Down Expand Up @@ -197,6 +206,13 @@ impl Expr {
let mapped = Box::new(map_internal(*expr, f));
Expr::Push(mapped)
}
Expr::Module(name, exprs) => {
let mut mapped = vec![];
for expr in exprs {
mapped.push(map_internal(expr, f))
}
Expr::Module(name, mapped)
}
expr => expr,
};

Expand All @@ -211,6 +227,7 @@ pub struct ExprTopDownIterator {
current: Option<Expr>,
next: Option<Expr>,
right_branches: Vec<Expr>,
module_context: Option<(String, IntoIter<Expr>)>,
}

impl ExprTopDownIterator {
Expand All @@ -219,13 +236,15 @@ impl ExprTopDownIterator {
current: None,
next: None,
right_branches: vec![],
module_context: None,
};
iter.iterate_expr(expr.clone());
iter
}

fn iterate_expr(&mut self, expr: Expr) {
self.current = Some(expr.clone());
dbg!("Running iter_expr on {:?} {:?}", &self.current, &self.next);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Tracking: should be removed or replaced with a log (or tracing) trace!

match expr {
Expr::Seq(lhs, rhs) => {
self.right_branches.push(*rhs);
Expand All @@ -235,6 +254,21 @@ impl ExprTopDownIterator {
self.right_branches.push(*rhs);
self.next = Some(*lhs);
}
Expr::Module(name, exprs) => {
if let Some((module_name, _)) = &self.module_context {
if name != *module_name {
// If we are in a different module, move to our module.
self.module_context = Some((name, exprs.into_iter()));
}
}

if let Some((_, exprs)) = &mut self.module_context {
match exprs.next() {
x @ Some(_) => self.next = x,
None => self.module_context = None,
}
}
}
Expr::PosPred(expr)
| Expr::NegPred(expr)
| Expr::Rep(expr)
Expand Down
4 changes: 3 additions & 1 deletion meta/src/grammar.pest
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,15 @@
// option. All files in the project carrying such notice may not be copied,
// modified, or distributed except according to those terms.

grammar_rules = _{ SOI ~ grammar_rule+ ~ EOI }
grammar_rules = _{ SOI ~ (grammar_rule | module)+ ~ EOI }

grammar_rule = {
identifier ~ assignment_operator ~ modifier? ~
opening_brace ~ expression ~ closing_brace
}

module = { "mod" ~ identifier ~ opening_brace ~ grammar_rule+ ~ closing_brace }

assignment_operator = { "=" }
opening_brace = { "{" }
closing_brace = { "}" }
Expand Down
5 changes: 5 additions & 0 deletions meta/src/optimizer/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,10 @@ fn rule_to_optimized_rule(rule: Rule) -> OptimizedRule {
Expr::Rep(expr) => OptimizedExpr::Rep(Box::new(to_optimized(*expr))),
Expr::Skip(strings) => OptimizedExpr::Skip(strings),
Expr::Push(expr) => OptimizedExpr::Push(Box::new(to_optimized(*expr))),
Expr::Module(name, exprs) => OptimizedExpr::Module(
name,
exprs.iter().map(|x| to_optimized(x.to_owned())).collect(),
),
Expr::RepOnce(_)
| Expr::RepExact(..)
| Expr::RepMin(..)
Expand Down Expand Up @@ -110,6 +114,7 @@ pub enum OptimizedExpr {
Skip(Vec<String>),
Push(Box<OptimizedExpr>),
RestoreOnErr(Box<OptimizedExpr>),
Module(String, Vec<OptimizedExpr>),
}

impl OptimizedExpr {
Expand Down
89 changes: 81 additions & 8 deletions meta/src/parser.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
// modified, or distributed except according to those terms.

use std::char;
use std::iter::Peekable;
use std::iter::{IntoIterator, Peekable};

use pest::error::{Error, ErrorVariant};
use pest::iterators::{Pair, Pairs};
Expand Down Expand Up @@ -58,7 +58,6 @@ impl<'i> ParserNode<'i> {
}

match node.expr {
// TODO: Use box syntax when it gets stabilized.
ParserExpr::PosPred(node) => {
filter_internal(*node, f, result);
}
Expand Down Expand Up @@ -97,6 +96,11 @@ impl<'i> ParserNode<'i> {
ParserExpr::Push(node) => {
filter_internal(*node, f, result);
}
ParserExpr::Module(_, nodes) => {
for node in nodes {
filter_internal(node, f, result);
}
}
_ => (),
}
}
Expand Down Expand Up @@ -128,6 +132,7 @@ pub enum ParserExpr<'i> {
RepMax(Box<ParserNode<'i>>, u32),
RepMinMax(Box<ParserNode<'i>>, u32, u32),
Push(Box<ParserNode<'i>>),
Module(String, Vec<ParserNode<'i>>),
}

fn convert_rule(rule: ParserRule) -> AstRule {
Expand All @@ -136,6 +141,16 @@ fn convert_rule(rule: ParserRule) -> AstRule {
AstRule { name, ty, expr }
}

fn rename_rule(rule: ParserRule, name: String) -> ParserRule {
ParserRule { name, ..rule }
}

fn prepend_rule(rule: ParserRule, prepend: String) -> ParserRule {
let name = rule.name;
let name = format!("{prepend}{name}");
ParserRule { name, ..rule }
}

fn convert_node(node: ParserNode) -> Expr {
match node.expr {
ParserExpr::Str(string) => Expr::Str(string),
Expand Down Expand Up @@ -163,9 +178,14 @@ fn convert_node(node: ParserNode) -> Expr {
Expr::RepMinMax(Box::new(convert_node(*node)), min, max)
}
ParserExpr::Push(node) => Expr::Push(Box::new(convert_node(*node))),
ParserExpr::Module(name, nodes) => Expr::Module(name, convert_nodes(nodes)),
}
}

pub fn convert_nodes<'a, I: IntoIterator<Item = ParserNode<'a>>>(nodes: I) -> Vec<Expr> {
nodes.into_iter().map(|x| convert_node(x)).collect()
}

pub fn consume_rules(pairs: Pairs<Rule>) -> Result<Vec<AstRule>, Vec<Error<Rule>>> {
let rules = consume_rules_with_spans(pairs)?;
let errors = validator::validate_ast(&rules);
Expand All @@ -177,12 +197,28 @@ pub fn consume_rules(pairs: Pairs<Rule>) -> Result<Vec<AstRule>, Vec<Error<Rule>
}

fn consume_rules_with_spans(pairs: Pairs<Rule>) -> Result<Vec<ParserRule>, Vec<Error<Rule>>> {
let mut rules: Vec<ParserRule> = vec![];

let climber = PrecClimber::new(vec![
Operator::new(Rule::choice_operator, Assoc::Left),
Operator::new(Rule::sequence_operator, Assoc::Left),
]);

pairs
let modules = pairs.clone().filter(|pair| pair.as_rule() == Rule::module);

for module in modules {
let mut pairs = module.into_inner();
let module_name = pairs.nth(1).unwrap().as_str(); // mod *x*
pairs.next().unwrap();
let x = consume_rules_with_spans(pairs)?;
let mut x = x
.iter()
.map(|rule| prepend_rule(rule.clone(), module_name.into()))
.collect();
rules.append(&mut x);
}

let mut pairs = pairs
.filter(|pair| pair.as_rule() == Rule::grammar_rule)
.map(|pair| {
let mut pairs = pair.into_inner().peekable();
Expand All @@ -206,16 +242,20 @@ fn consume_rules_with_spans(pairs: Pairs<Rule>) -> Result<Vec<ParserRule>, Vec<E

pairs.next().unwrap(); // opening_brace

let node = consume_expr(pairs.next().unwrap().into_inner().peekable(), &climber)?;
let node =
consume_expr(pairs.next().unwrap().into_inner().peekable(), &climber).unwrap();

Ok(ParserRule {
ParserRule {
name,
span,
ty,
node,
})
}
})
.collect()
.collect();

rules.append(&mut pairs);
Ok(rules)
}

fn consume_expr<'i>(
Expand Down Expand Up @@ -1013,12 +1053,45 @@ mod tests {
};
}

#[test]
fn module() {
parses_to! {
parser: PestParser,
input: r#"mod x {
y = { "y" }
}"#,
rule: Rule::grammar_rules,
tokens: [
module(0, 49, [
identifier(4, 5),
opening_brace(6, 7),
grammar_rule(24, 35, [
identifier(24, 25),
assignment_operator(26, 27),
opening_brace(28, 29),
expression(30, 34, [
term(30, 34, [
string(30, 33, [
quote(30, 31),
inner_str(31, 32),
quote(32, 33),
])
])
]),
closing_brace(34, 35),
]),
closing_brace(48, 49),
])
]
};
}

#[test]
fn wrong_identifier() {
fails_with! {
parser: PestParser,
input: "0",
rule: Rule::grammar_rules,
rule: Rule::identifier,
positives: vec![Rule::identifier],
negatives: vec![],
pos: 0
Expand Down
4 changes: 3 additions & 1 deletion meta/src/validator.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,9 @@ pub fn validate_pairs(pairs: Pairs<Rule>) -> Result<Vec<&str>, Vec<Error<Rule>>>
let definitions: Vec<_> = pairs
.clone()
.filter(|pair| pair.as_rule() == Rule::grammar_rule)
.map(|pair| pair.into_inner().next().unwrap().as_span())
.map(|pair| pair.into_inner().next().unwrap())
.filter(|pair| pair.as_rule() != Rule::module)
.map(|pair| pair.as_span())
.collect();
let called_rules: Vec<_> = pairs
.clone()
Expand Down
9 changes: 8 additions & 1 deletion vm/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,7 @@ impl Vm {
fn parse_expr<'a, 'i>(
&'a self,
expr: &'a OptimizedExpr,
state: Box<ParserState<'i, &'a str>>,
mut state: Box<ParserState<'i, &'a str>>,
) -> ParseResult<Box<ParserState<'i, &'a str>>> {
match *expr {
OptimizedExpr::Str(ref string) => state.match_string(string),
Expand Down Expand Up @@ -193,6 +193,13 @@ impl Vm {
.map(|state| state.as_str())
.collect::<Vec<&str>>(),
),
OptimizedExpr::Module(_, ref exprs) => {
for expr in exprs {
state = self.parse_expr(expr, state)?;
}

Ok(state)
}
OptimizedExpr::RestoreOnErr(ref expr) => {
state.restore_on_err(|state| self.parse_expr(expr, state))
}
Expand Down