fix: restrict the factorizer case to only atomic and compoundatomic r…

…ules (#766)

* fix: restrict the factorizer case to only atomic and compoundatomic rules

closes #762

the full fix would require a bigger overhaul / refactoring of the optimizer

* add alloc

Co-authored-by: Tomas Tauber <[email protected]>

debugger/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "pest_debugger"
 description = "pest grammar debugger"
-version = "2.5.2"
+version = "2.5.3"
 edition = "2021"
 authors = ["Dragoș Tiselice <[email protected]>", "Tomas Tauber <[email protected]>"]
 homepage = "https://pest.rs/"
@@ -14,9 +14,9 @@ readme = "_README.md"
 rust-version = "1.56"
 
 [dependencies]
-pest = { path = "../pest", version = "2.5.2" }
-pest_meta = { path = "../meta", version = "2.5.2" }
-pest_vm = { path = "../vm", version = "2.5.2" }
+pest = { path = "../pest", version = "2.5.3" }
+pest_meta = { path = "../meta", version = "2.5.3" }
+pest_vm = { path = "../vm", version = "2.5.3" }
 rustyline = "10"
 thiserror = "1"
 

derive/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "pest_derive"
 description = "pest's derive macro"
-version = "2.5.2"
+version = "2.5.3"
 edition = "2021"
 authors = ["Dragoș Tiselice <[email protected]>"]
 homepage = "https://pest.rs/"
@@ -23,5 +23,5 @@ std = ["pest/std", "pest_generator/std"]
 
 [dependencies]
 # for tests, included transitively anyway
-pest = { path = "../pest", version = "2.5.2", default-features = false }
-pest_generator = { path = "../generator", version = "2.5.2", default-features = false }
+pest = { path = "../pest", version = "2.5.3", default-features = false }
+pest_generator = { path = "../generator", version = "2.5.3", default-features = false }

derive/tests/implicit.pest

@@ -0,0 +1,14 @@
+program = _{ SOI ~ implicit ~ EOI  }
+implicit= ${ or ~ (WHITESPACE+ ~ or )* }
+
+or  = !{ and ~ (or_op ~ and)+ | and }
+and = { comp ~ (and_op ~ comp)+ | comp }
+comp = { array ~ eq_op ~ array | array }
+
+array = ${ term }
+
+term = _{ ASCII_ALPHANUMERIC+ }
+or_op = { "||" }
+and_op = { "&&" }
+eq_op = { "=" }
+WHITESPACE = _{ " " | "\t" | NEWLINE }

derive/tests/implicit.rs

@@ -0,0 +1,25 @@
+// Licensed under the Apache License, Version 2.0
+// <LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0> or the MIT
+// license <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. All files in the project carrying such notice may not be copied,
+// modified, or distributed except according to those terms.
+
+#![cfg_attr(not(feature = "std"), no_std)]
+extern crate alloc;
+extern crate pest;
+extern crate pest_derive;
+
+use pest::Parser;
+use pest_derive::Parser;
+
+#[derive(Parser)]
+#[grammar = "../tests/implicit.pest"]
+struct TestImplicitParser;
+
+#[test]
+fn test_implicit_whitespace() {
+    // this failed to parse due to a bug in the optimizer
+    // see: https://github.com/pest-parser/pest/issues/762#issuecomment-1375374868
+    let successful_parse = TestImplicitParser::parse(Rule::program, "a a");
+    assert!(successful_parse.is_ok());
+}

generator/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "pest_generator"
 description = "pest code generator"
-version = "2.5.2"
+version = "2.5.3"
 edition = "2021"
 authors = ["Dragoș Tiselice <[email protected]>"]
 homepage = "https://pest.rs/"
@@ -18,8 +18,8 @@ default = ["std"]
 std = ["pest/std"]
 
 [dependencies]
-pest = { path = "../pest", version = "2.5.2", default-features = false }
-pest_meta = { path = "../meta", version = "2.5.2" }
+pest = { path = "../pest", version = "2.5.3", default-features = false }
+pest_meta = { path = "../meta", version = "2.5.3" }
 proc-macro2 = "1.0"
 quote = "1.0"
 syn = "1.0"

grammars/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "pest_grammars"
 description = "pest popular grammar implementations"
-version = "2.5.2"
+version = "2.5.3"
 edition = "2021"
 authors = ["Dragoș Tiselice <[email protected]>"]
 homepage = "https://pest.rs/"
@@ -14,8 +14,8 @@ readme = "_README.md"
 rust-version = "1.56"
 
 [dependencies]
-pest = { path = "../pest", version = "2.5.2" }
-pest_derive = { path = "../derive", version = "2.5.2" }
+pest = { path = "../pest", version = "2.5.3" }
+pest_derive = { path = "../derive", version = "2.5.3" }
 
 [dev-dependencies]
 criterion = "0.3"

meta/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "pest_meta"
 description = "pest meta language parser and validator"
-version = "2.5.2"
+version = "2.5.3"
 edition = "2021"
 authors = ["Dragoș Tiselice <[email protected]>"]
 homepage = "https://pest.rs/"
@@ -16,7 +16,7 @@ include = ["Cargo.toml", "src/**/*", "src/grammar.rs", "_README.md", "LICENSE-*"
 rust-version = "1.56"
 
 [dependencies]
-pest = { path = "../pest", version = "2.5.2" }
+pest = { path = "../pest", version = "2.5.3" }
 once_cell = "1.8.0"
 
 [build-dependencies]

meta/src/optimizer/factorizer.rs

@@ -26,7 +26,11 @@ pub fn factor(rule: Rule) -> Rule {
                         }
                     }
                     // Converts `(rule ~ rest) | rule` to `rule ~ rest?`, avoiding trying to match `rule` twice.
-                    (Expr::Seq(l1, l2), r) => {
+                    // This is only done for atomic rules, because other rule types have implicit whitespaces.
+                    // FIXME: "desugar" implicit whitespace rules before applying any optimizations
+                    (Expr::Seq(l1, l2), r)
+                        if matches!(ty, RuleType::Atomic | RuleType::CompoundAtomic) =>
+                    {
                         if *l1 == r {
                             Expr::Seq(l1, Box::new(Expr::Opt(l2)))
                         } else {

meta/src/optimizer/mod.rs

@@ -543,7 +543,7 @@ mod tests {
             use crate::ast::Expr::*;
             vec![Rule {
                 name: "rule".to_owned(),
-                ty: RuleType::Silent,
+                ty: RuleType::Atomic,
                 expr: box_tree!(Choice(
                     Seq(Ident(String::from("a")), Ident(String::from("b"))),
                     Ident(String::from("a"))
@@ -554,7 +554,7 @@ mod tests {
             use crate::optimizer::OptimizedExpr::*;
             vec![OptimizedRule {
                 name: "rule".to_owned(),
-                ty: RuleType::Silent,
+                ty: RuleType::Atomic,
                 expr: box_tree!(Seq(Ident(String::from("a")), Opt(Ident(String::from("b"))))),
             }]
         };

pest/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "pest"
 description = "The Elegant Parser"
-version = "2.5.2"
+version = "2.5.3"
 edition = "2021"
 authors = ["Dragoș Tiselice <[email protected]>"]
 homepage = "https://pest.rs/"

vm/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "pest_vm"
 description = "pest grammar virtual machine"
-version = "2.5.2"
+version = "2.5.3"
 edition = "2021"
 authors = ["Dragoș Tiselice <[email protected]>"]
 homepage = "https://pest.rs/"
@@ -14,5 +14,5 @@ readme = "_README.md"
 rust-version = "1.56"
 
 [dependencies]
-pest = { path = "../pest", version = "2.5.2" }
-pest_meta = { path = "../meta", version = "2.5.2" }
+pest = { path = "../pest", version = "2.5.3" }
+pest_meta = { path = "../meta", version = "2.5.3" }