Added SupplementaryCharTok

parsley/shared/src/main/scala/parsley/character.scala

@@ -125,7 +125,7 @@ object character {
       */
     private [parsley] def charUtf16(c: Int): Parsley[Int] = { //TODO: release along with the utf combinators
         if (Character.isBmpCodePoint(c)) char(c.toChar) #> c
-        else attempt(string(Character.toChars(c).mkString)) #> c
+        else new Parsley(new singletons.SupplementaryCharTok(c, NotConfigured))
     }
 
     /** This combinator tries to parse a single character from the input that matches the given predicate.

parsley/shared/src/main/scala/parsley/internal/deepembedding/backend/AlternativeEmbedding.scala

@@ -227,7 +227,8 @@ private [backend] object Choice {
 
     @tailrec private def tablable(p: StrictParsley[_], backtracks: Boolean): Option[(Char, Option[ExpectItem], Int, Boolean)] = p match {
         // CODO: Numeric parsers by leading digit (This one would require changing the foldTablified function a bit)
-        case ct@CharTok(d)                       => Some((d, ct.expected.asExpectItem(d), 1, backtracks))
+        case ct@CharTok(c)                       => Some((c, ct.expected.asExpectItem(c), 1, backtracks))
+        case ct@SupplementaryCharTok(c)          => Some((Character.highSurrogate(c), ct.expected.asExpectItem(Character.toChars(c).mkString), 1, backtracks))
         case st@StringTok(s)                     => Some((s.head, st.expected.asExpectItem(s), s.codePointCount(0, s.length), backtracks))
         //case op@MaxOp(o)                         => Some((o.head, Some(Desc(o)), o.size, backtracks))
         //case _: StringLiteral | RawStringLiteral => Some(('"', Some(Desc("string")), 1, backtracks))

parsley/shared/src/main/scala/parsley/internal/deepembedding/backend/ErrorEmbedding.scala

@@ -17,6 +17,7 @@ private [deepembedding] final class ErrorLabel[A](val p: StrictParsley[A], priva
     override def handlerLabel(state: CodeGenState): Int = state.getLabelForRelabelError(label)
     final override def optimise: StrictParsley[A] = p match {
         case ct@CharTok(c) if ct.expected ne Hidden => new CharTok(c, Label(label)).asInstanceOf[StrictParsley[A]]
+        case ct@SupplementaryCharTok(c) if ct.expected ne Hidden => new SupplementaryCharTok(c, Label(label)).asInstanceOf[StrictParsley[A]]
         case st@StringTok(s) if st.expected ne Hidden => new StringTok(s, Label(label)).asInstanceOf[StrictParsley[A]]
         case sat@Satisfy(f) if sat.expected ne Hidden => new Satisfy(f, Label(label)).asInstanceOf[StrictParsley[A]]
         case sat@UniSatisfy(f) if sat.expected ne Hidden => new UniSatisfy(f, Label(label)).asInstanceOf[StrictParsley[A]]

parsley/shared/src/main/scala/parsley/internal/deepembedding/backend/SequenceEmbedding.scala

@@ -73,6 +73,7 @@ private [deepembedding] final class <*>[A, B](var left: StrictParsley[A => B], v
         // pure f <*> p = f <$> p
         case Pure(f) => right match {
             case ct@CharTok(c) => result(instrs += instructions.CharTokFastPerform[Char, B](c, f.asInstanceOf[Char => B], ct.expected))
+            case ct@SupplementaryCharTok(c) => result(instrs += instructions.SupplementaryCharTokFastPerform[Int, B](c, f.asInstanceOf[Int => B], ct.expected))
             case st@StringTok(s) => result(instrs += instructions.StringTokFastPerform(s, f.asInstanceOf[String => B], st.expected))
             case _ =>
                 suspend(right.codeGen[Cont, R]) |>

parsley/shared/src/main/scala/parsley/internal/deepembedding/singletons/IntrinsicEmbedding.scala

@@ -13,14 +13,21 @@ private [parsley] final class CharTok(private [CharTok] val c: Char, val expecte
     // $COVERAGE-OFF$
     override def pretty: String = s"char($c)"
     // $COVERAGE-ON$
-    override def instr: instructions.Instr = instructions.CharTok(c, expected)
+    override def instr: instructions.Instr = new instructions.CharTok(c, expected)
+}
+
+private [parsley] final class SupplementaryCharTok(private [SupplementaryCharTok] val codepoint: Int, val expected: LabelConfig) extends Singleton[Int] {
+    // $COVERAGE-OFF$
+    override def pretty: String = s"char(${Character.toChars(codepoint).mkString})"
+    // $COVERAGE-ON$
+    override def instr: instructions.Instr = new instructions.SupplementaryCharTok(codepoint, expected)
 }
 
 private [parsley] final class StringTok(private [StringTok] val s: String, val expected: LabelConfig) extends Singleton[String] {
     // $COVERAGE-OFF$
     override def pretty: String = s"string($s)"
     // $COVERAGE-ON$
-    override def instr: instructions.Instr = instructions.StringTok(s, expected)
+    override def instr: instructions.Instr = new instructions.StringTok(s, expected)
 }
 
 private [parsley] object Eof extends Singleton[Unit] {
@@ -47,6 +54,9 @@ private [parsley] final class Modify[S](val reg: Reg[S], f: S => S) extends Sing
 private [deepembedding] object CharTok {
     def unapply(self: CharTok): Option[Char] = Some(self.c)
 }
+private [deepembedding] object SupplementaryCharTok {
+    def unapply(self: SupplementaryCharTok): Option[Int] = Some(self.codepoint)
+}
 private [deepembedding] object StringTok {
     def unapply(self: StringTok): Option[String] = Some(self.s)
 }

parsley/shared/src/main/scala/parsley/internal/machine/instructions/IntrinsicInstrs.scala

@@ -42,6 +42,8 @@ private [internal] object Lift3 {
 }
 
 private [internal] class CharTok(c: Char, x: Any, errorItem: Option[ExpectItem]) extends Instr {
+    def this(c: Char, x: Any, expected: LabelConfig) = this(c, x, expected.asExpectItem(s"$c"))
+    def this(c: Char, expected: LabelConfig) = this(c, c, expected)
     override def apply(ctx: Context): Unit = {
         ensureRegularInstruction(ctx)
         if (ctx.moreInput && ctx.peekChar == c) {
@@ -51,11 +53,38 @@ private [internal] class CharTok(c: Char, x: Any, errorItem: Option[ExpectItem])
         else ctx.expectedFail(errorItem, unexpectedWidth = 1)
     }
     // $COVERAGE-OFF$
-    override def toString: String = if (x == c) s"Chr($c)" else s"ChrPerform($c, $x)"
+    override def toString: String = if (x == c) s"Chr($c)" else s"ChrExchange($c, $x)"
+    // $COVERAGE-ON$
+}
+
+private [internal] class SupplementaryCharTok(codepoint: Int, x: Any, errorItem: Option[ExpectItem]) extends Instr {
+    def this(codepoint: Int, x: Any, expected: LabelConfig) = this(codepoint, x, expected.asExpectItem(Character.toChars(codepoint).mkString))
+    def this(codepoint: Int, expected: LabelConfig) = this(codepoint, codepoint, expected)
+
+    assert(Character.isSupplementaryCodePoint(codepoint), "SupplementaryCharTok should only be used for supplementary code points")
+    val h = Character.highSurrogate(codepoint)
+    val l = Character.lowSurrogate(codepoint)
+    override def apply(ctx: Context): Unit = {
+        ensureRegularInstruction(ctx)
+        if (ctx.moreInput(2) && ctx.peekChar(0) == h && ctx.peekChar(1) == l) {
+            // not going to be a tab or newline
+            ctx.offset += 2
+            ctx.col += 1
+            ctx.pushAndContinue(x)
+        }
+        else ctx.expectedFail(errorItem, unexpectedWidth = 1)
+    }
+    // $COVERAGE-OFF$
+    override def toString: String =
+        if (x == codepoint) s"SupplementaryChr($h$l)"
+        else s"SupplementaryChrExchange($h$l, $x)"
     // $COVERAGE-ON$
 }
 
 private [internal] final class StringTok(s: String, x: Any, errorItem: Option[ExpectItem]) extends Instr {
+    def this(s: String, x: Any, expected: LabelConfig) = this(s, x,  expected.asExpectItem(s))
+    def this(s: String, expected: LabelConfig) = this(s, s, expected)
+
     private [this] val sz = s.length
     private [this] val codePointLength = s.codePointCount(0, sz)
 
@@ -331,15 +360,7 @@ private [internal] final class SwapAndPut(reg: Int) extends Instr {
 }
 
 // Companion Objects
-private [internal] object CharTok {
-    def apply(c: Char, expected: LabelConfig): CharTok = apply(c, c, expected)
-    def apply(c: Char, x: Any, expected: LabelConfig): CharTok = new CharTok(c, x, expected.asExpectItem(s"$c"))
-}
-
 private [internal] object StringTok {
-    def apply(s: String, expected: LabelConfig): StringTok = apply(s, s, expected)
-    def apply(s: String, x: Any, expected: LabelConfig): StringTok = new StringTok(s, x,  expected.asExpectItem(s))
-
     private [StringTok] abstract class Adjust {
         private [StringTok] def tab: Adjust
         private [StringTok] def next(): Unit
@@ -382,9 +403,13 @@ private [internal] object StringTok {
 }
 
 private [internal] object CharTokFastPerform {
-    def apply[A >: Char, B](c: Char, f: A => B, expected: LabelConfig): CharTok = CharTok(c, f(c), expected)
+    def apply[A >: Char, B](c: Char, f: A => B, expected: LabelConfig): CharTok = new CharTok(c, f(c), expected)
+}
+
+private [internal] object SupplementaryCharTokFastPerform {
+    def apply[A >: Int, B](c: Int, f: A => B, expected: LabelConfig): SupplementaryCharTok = new SupplementaryCharTok(c, f(c), expected)
 }
 
 private [internal] object StringTokFastPerform {
-    def apply(s: String, f: String => Any, expected: LabelConfig): StringTok = StringTok(s, f(s), expected)
+    def apply(s: String, f: String => Any, expected: LabelConfig): StringTok = new StringTok(s, f(s), expected)
 }