Optimised softOperator and added Trie implementation

build.sbt

@@ -69,7 +69,8 @@ lazy val parsley = crossProject(JSPlatform, JVMPlatform, NativePlatform)
     name := projectName,
 
     libraryDependencies ++= Seq(
-        "org.scalatest" %%% "scalatest" % "3.2.14" % Test,
+        "org.scalatest" %%% "scalatest" % "3.2.15" % Test,
+        "org.scalatestplus" %%% "scalacheck-1-17" % "3.2.15.0" % Test,
     ),
 
     Test / testOptions += Tests.Argument(TestFrameworks.ScalaTest, "-oI"),

parsley/shared/src/main/scala/parsley/internal/collection/immutable/Trie.scala

@@ -0,0 +1,48 @@
+package parsley.internal.collection.immutable
+
+import scala.annotation.tailrec
+import scala.collection.immutable.IntMap
+
+private [parsley] class Trie(private val present: Boolean, children: IntMap[Trie]) {
+    def contains(key: String): Boolean = suffixes(key).present/*contains(key, 0, key.length)
+    @tailrec private def contains(key: String, idx: Int, sz: Int): Boolean = {
+        if (idx == sz) present
+        else childAt(key, idx) match {
+            case None => false
+            case Some(t) => t.contains(key, idx + 1, sz)
+        }
+    }*/
+
+    def isEmpty: Boolean = this eq Trie.empty
+    def nonEmpty: Boolean = !isEmpty
+
+    def suffixes(key: Char): Trie = children.getOrElse(key.toInt, Trie.empty)
+    def suffixes(key: String): Trie = suffixes(key, 0, key.length)
+    @tailrec private def suffixes(key: String, idx: Int, sz: Int): Trie = {
+        if (idx == sz) this
+        else childAt(key, idx) match {
+            case None => Trie.empty
+            case Some(t) => t.suffixes(key, idx + 1, sz)
+        }
+    }
+
+    def incl(key: String): Trie = incl(key, 0, key.length)
+    private def incl(key: String, idx: Int, sz: Int): Trie = {
+        if (idx == sz && present) this
+        else if (idx == sz) new Trie(present = true, children)
+        else childAt(key, idx) match {
+            case None => new Trie(present, children.updated(key.charAt(idx).toInt, Trie.empty.incl(key, idx + 1, sz)))
+            case Some(t) =>
+                val newT = t.incl(key, idx + 1, sz)
+                if (t eq newT) this
+                else new Trie(present, children.updated(key.charAt(idx).toInt, newT))
+        }
+    }
+
+    private def childAt(key: String, idx: Int) = children.get(key.charAt(idx).toInt)
+}
+private [parsley] object Trie {
+    val empty = new Trie(present = false, IntMap.empty)
+
+    def apply(strs: Iterable[String]): Trie = strs.foldLeft(empty)(_.incl(_))
+}

parsley/shared/src/main/scala/parsley/internal/deepembedding/singletons/token/SymbolEmbedding.scala

@@ -6,6 +6,7 @@ package parsley.internal.deepembedding.singletons.token
 import parsley.token.errors.LabelConfig
 import parsley.token.predicate.CharPredicate
 
+import parsley.internal.collection.immutable.Trie
 import parsley.internal.deepembedding.singletons.Singleton
 import parsley.internal.machine.instructions
 
@@ -17,6 +18,14 @@ private [parsley] final class SoftKeyword(private [SoftKeyword] val specific: St
     override def instr: instructions.Instr = new instructions.token.SoftKeyword(specific, letter, caseSensitive, expected, expectedEnd)
 }
 
+private [parsley] final class SoftOperator(private [SoftOperator] val specific: String, letter: CharPredicate, ops: Trie,
+                                           expected: LabelConfig, expectedEnd: String) extends Singleton[Unit] {
+    // $COVERAGE-OFF$
+    override def pretty: String = s"softOperator($specific)"
+    // $COVERAGE-ON$
+    override def instr: instructions.Instr = new instructions.token.SoftOperator(specific, letter, ops, expected, expectedEnd)
+}
+
 /*
 private [parsley] final class MaxOp(private [MaxOp] val operator: String, ops: Set[String]) extends Singleton[Unit] {
     // $COVERAGE-OFF$
@@ -30,4 +39,7 @@ private [parsley] final class MaxOp(private [MaxOp] val operator: String, ops: S
 private [deepembedding] object SoftKeyword {
     def unapply(self: SoftKeyword): Some[String] = Some(self.specific)
 }
+private [deepembedding] object SoftOperator {
+    def unapply(self: SoftOperator): Some[String] = Some(self.specific)
+}
 // $COVERAGE-ON$

parsley/shared/src/main/scala/parsley/internal/machine/instructions/token/SymbolInstrs.scala

@@ -3,26 +3,26 @@
  */
 package parsley.internal.machine.instructions.token
 
+import scala.annotation.tailrec
+
 import parsley.token.errors.LabelConfig
 import parsley.token.predicate
 
 import parsley.internal.errors.ExpectDesc
 import parsley.internal.machine.Context
 import parsley.internal.machine.XAssert._
 import parsley.internal.machine.instructions.Instr
+import parsley.internal.collection.immutable.Trie
 
-private [internal] final class SoftKeyword(
-        specific: String, letter: CharPredicate, caseSensitive: Boolean, expected: Option[ExpectDesc], expectedEnd: Option[ExpectDesc]) extends Instr {
-    def this(specific: String, letter: predicate.CharPredicate, caseSensitive: Boolean, expected: LabelConfig, expectedEnd: String) = {
-        this(if (caseSensitive) specific else specific.toLowerCase,
-             letter.asInternalPredicate,
-             caseSensitive,
-             expected.asExpectDesc, Some(new ExpectDesc(expectedEnd)))
-    }
-
+private [token] abstract class Specific extends Instr {
+    protected val specific: String
+    protected val caseSensitive: Boolean
+    protected val expected: Option[ExpectDesc]
     private [this] final val strsz = specific.length
     private [this] final val numCodePoints = specific.codePointCount(0, strsz)
 
+    protected def postprocess(ctx: Context): Unit
+
     final override def apply(ctx: Context): Unit = {
         ensureRegularInstruction(ctx)
         if (ctx.moreInput(strsz)) {
@@ -32,23 +32,12 @@ private [internal] final class SoftKeyword(
         else ctx.expectedFail(expected, numCodePoints)
     }
 
-    private def postprocess(ctx: Context): Unit = {
-        if (letter.peek(ctx)) {
-            ctx.expectedFail(expectedEnd, unexpectedWidth = 1) //This should only report a single token
-            ctx.restoreState()
-        }
-        else {
-            ctx.states = ctx.states.tail
-            ctx.pushAndContinue(())
-        }
-    }
-
-    val readCharCaseHandledBMP = {
+    private val readCharCaseHandledBMP = {
         if (caseSensitive) (ctx: Context) => ctx.peekChar
         else (ctx: Context) => ctx.peekChar.toLower
     }
 
-    val readCharCaseHandledSupplementary = {
+    private val readCharCaseHandledSupplementary = {
         if (caseSensitive) (ctx: Context) => Character.toCodePoint(ctx.peekChar(0), ctx.peekChar(1))
         else (ctx: Context) => Character.toLowerCase(Character.toCodePoint(ctx.peekChar(0), ctx.peekChar(1)))
     }
@@ -71,8 +60,61 @@ private [internal] final class SoftKeyword(
         }
         else postprocess(ctx)
     }
+}
+
+private [internal] final class SoftKeyword(protected val specific: String, letter: CharPredicate, protected val caseSensitive: Boolean,
+                                           protected val expected: Option[ExpectDesc], expectedEnd: Option[ExpectDesc]) extends Specific {
+    def this(specific: String, letter: predicate.CharPredicate, caseSensitive: Boolean, expected: LabelConfig, expectedEnd: String) = {
+        this(if (caseSensitive) specific else specific.toLowerCase,
+             letter.asInternalPredicate,
+             caseSensitive,
+             expected.asExpectDesc, Some(new ExpectDesc(expectedEnd)))
+    }
+
+    protected def postprocess(ctx: Context): Unit = {
+        if (letter.peek(ctx)) {
+            ctx.expectedFail(expectedEnd, unexpectedWidth = 1) //This should only report a single token
+            ctx.restoreState()
+        }
+        else {
+            ctx.states = ctx.states.tail
+            ctx.pushAndContinue(())
+        }
+    }
 
     // $COVERAGE-OFF$
     override def toString: String = s"SoftKeyword($specific)"
     // $COVERAGE-ON$
 }
+
+private [internal] final class SoftOperator(protected val specific: String, letter: CharPredicate, ops: Trie, protected val expected: Option[ExpectDesc], expectedEnd: Option[ExpectDesc]) extends Specific {
+    def this(specific: String, letter: predicate.CharPredicate, ops: Trie, expected: LabelConfig, expectedEnd: String) = {
+        this(specific, letter.asInternalPredicate, ops, expected.asExpectDesc, Some(new ExpectDesc(expectedEnd)))
+    }
+    protected val caseSensitive = true
+    private val ends = ops.suffixes(specific)
+
+    // returns true if an end could be parsed from this point
+    @tailrec private def checkEnds(ctx: Context, ends: Trie, off: Int): Boolean = {
+        if (ends.nonEmpty && ctx.moreInput(off + 1)) {
+            val endsOfNext = ends.suffixes(ctx.peekChar(off))
+            endsOfNext.contains("") || checkEnds(ctx, endsOfNext, off + 1)
+        }
+        else false
+    }
+
+    protected def postprocess(ctx: Context): Unit = {
+        if (letter.peek(ctx) || checkEnds(ctx, ends, off = 0)) {
+            ctx.expectedFail(expectedEnd, unexpectedWidth = 1) //This should only report a single token
+            ctx.restoreState()
+        }
+        else {
+            ctx.states = ctx.states.tail
+            ctx.pushAndContinue(())
+        }
+    }
+
+    // $COVERAGE-OFF$
+    override def toString: String = s"SoftOperator($specific)"
+    // $COVERAGE-ON$
+}

parsley/shared/src/main/scala/parsley/token/descriptions/SymbolDesc.scala

@@ -3,6 +3,8 @@
  */
 package parsley.token.descriptions
 
+import parsley.internal.collection.immutable.Trie
+
 /** This class describes how symbols (textual literals in a BNF) should be
   * processed lexically.
   *
@@ -15,6 +17,7 @@ final case class SymbolDesc (hardKeywords: Set[String],
                              hardOperators: Set[String],
                              caseSensitive: Boolean) {
     require((hardKeywords & hardOperators).isEmpty, "there cannot be an intersection between keywords and operators")
+    private [parsley] val hardOperatorsTrie = Trie(hardOperators)
     private [parsley] def isReservedName(name: String): Boolean =
         theReservedNames.contains(if (caseSensitive) name else name.toLowerCase)
     private lazy val theReservedNames =  if (caseSensitive) hardKeywords else hardKeywords.map(_.toLowerCase)

parsley/shared/src/main/scala/parsley/token/symbol/ConcreteSymbol.scala

@@ -3,9 +3,8 @@
  */
 package parsley.token.symbol
 
-import parsley.Parsley, Parsley.{attempt, notFollowedBy}
-import parsley.character.{char, string, strings}
-import parsley.errors.combinator.ErrorMethods
+import parsley.Parsley, Parsley.attempt
+import parsley.character.{char, string}
 import parsley.token.descriptions.{NameDesc, SymbolDesc}
 import parsley.token.errors.ErrorConfig
 
@@ -49,14 +48,17 @@ private [token] class ConcreteSymbol(nameDesc: NameDesc, symbolDesc: SymbolDesc,
     */
 
     override def softKeyword(name: String): Parsley[Unit] = {
+        require(name.nonEmpty, "Keywords may not be empty strings")
         new Parsley(new token.SoftKeyword(name, nameDesc.identifierLetter, symbolDesc.caseSensitive,
                                           err.labelSymbolKeyword(name), err.labelSymbolEndOfKeyword(name)))
     }
 
-    private lazy val opLetter = nameDesc.operatorLetter.toNative
+    //private lazy val opLetter = nameDesc.operatorLetter.toNative
     override def softOperator(name: String): Parsley[Unit] = {
         require(name.nonEmpty, "Operators may not be empty strings")
-        val ends = symbolDesc.hardOperators.collect {
+        new Parsley(new token.SoftOperator(name, nameDesc.operatorLetter, symbolDesc.hardOperatorsTrie,
+                                           err.labelSymbolOperator(name), err.labelSymbolEndOfOperator(name)))
+        /*val ends = symbolDesc.hardOperators.collect {
             case op if op.startsWith(name) && op != name => op.substring(name.length)
         }.toList
         ends match {
@@ -68,6 +70,6 @@ private [token] class ConcreteSymbol(nameDesc: NameDesc, symbolDesc: SymbolDesc,
                 err.labelSymbolOperator(name)(string(name)) *>
                 notFollowedBy(opLetter <|> strings(end, ends: _*)).label(err.labelSymbolEndOfOperator(name))
             }
-        }
+        }*/
     }
 }

parsley/shared/src/test/scala/parsley/internal/collection/immutable/TrieSpec.scala

@@ -0,0 +1,28 @@
+package parsley.internal.collection.immutable
+
+import org.scalatest.propspec.AnyPropSpec
+import org.scalatest.matchers._
+import org.scalatestplus.scalacheck.ScalaCheckPropertyChecks
+
+class TrieSpec extends AnyPropSpec with ScalaCheckPropertyChecks with should.Matchers {
+    property("a Trie constructed from a set should contain all of its elements") {
+        forAll { (set: Set[String]) =>
+            val t = Trie(set)
+            set.forall(t.contains)
+            for (key <- set) {
+                t.contains(key) shouldBe true
+            }
+        }
+    }
+
+    property("a Trie constructed from a set should not contain extra keys") {
+        forAll { (set: Set[String]) =>
+            val t = Trie(set)
+            forAll { (str: String) =>
+                whenever(!set.contains(str)) {
+                    t.contains(str) shouldBe false
+                }
+            }
+        }
+    }
+}