Common Parser and AST for Tokens

compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/AccessModifierParser.scala

@@ -20,14 +20,14 @@ import fastparse._
 import fastparse.NoWhitespace.noWhitespaceImplicit
 import org.alephium.ralph.lsp.access.compiler.message.SourceIndexExtra.range
 import org.alephium.ralph.lsp.access.compiler.parser.soft.CommonParser._
-import org.alephium.ralph.lsp.access.compiler.parser.soft.ast.SoftAST
+import org.alephium.ralph.lsp.access.compiler.parser.soft.ast.{SoftAST, Token}
 
 private object AccessModifierParser {
 
   def parseOrFail[Unknown: P]: P[SoftAST.AccessModifier] =
     P {
       Index ~
-        TokenParser.PubOrFail ~
+        TokenParser.parseOrFail(Token.Pub) ~
         spaceOrFail.? ~
         Index
     } map {

compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/AnnotationParser.scala

@@ -20,7 +20,7 @@ import fastparse._
 import fastparse.NoWhitespace.noWhitespaceImplicit
 import org.alephium.ralph.lsp.access.compiler.message.SourceIndexExtra.range
 import org.alephium.ralph.lsp.access.compiler.parser.soft.CommonParser._
-import org.alephium.ralph.lsp.access.compiler.parser.soft.ast.SoftAST
+import org.alephium.ralph.lsp.access.compiler.parser.soft.ast.{SoftAST, Token}
 
 private object AnnotationParser {
 
@@ -39,7 +39,7 @@ private object AnnotationParser {
   def parseOrFail[Unknown: P]: P[SoftAST.Annotation] =
     P {
       Index ~
-        TokenParser.AtOrFail ~
+        TokenParser.parseOrFail(Token.At) ~
         spaceOrFail.? ~
         identifier ~
         spaceOrFail.? ~

compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/AssignmentAccessModifierParser.scala

@@ -20,21 +20,21 @@ import fastparse._
 import fastparse.NoWhitespace.noWhitespaceImplicit
 import org.alephium.ralph.lsp.access.compiler.message.SourceIndexExtra.range
 import org.alephium.ralph.lsp.access.compiler.parser.soft.CommonParser._
-import org.alephium.ralph.lsp.access.compiler.parser.soft.ast.SoftAST
+import org.alephium.ralph.lsp.access.compiler.parser.soft.ast.{SoftAST, Token}
 
 private object AssignmentAccessModifierParser {
 
   def parseOrFail[Unknown: P]: P[SoftAST.AssignmentAccessModifier] =
     P {
       Index ~
-        (TokenParser.LetOrFail | TokenParser.MutOrFail) ~
+        (TokenParser.parseOrFail(Token.Let) | TokenParser.parseOrFail(Token.Mut)) ~
         spaceOrFail.? ~
         Index
     } map {
-      case (from, doubleForwardSlash, space, to) =>
+      case (from, dataAssignment, space, to) =>
         SoftAST.AssignmentAccessModifier(
           index = range(from, to),
-          doubleForwardSlash,
+          token = dataAssignment,
           postTokenSpace = space
         )
     }

compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/AssignmentParser.scala

@@ -4,7 +4,7 @@ import fastparse._
 import fastparse.NoWhitespace.noWhitespaceImplicit
 import org.alephium.ralph.lsp.access.compiler.message.SourceIndexExtra.range
 import org.alephium.ralph.lsp.access.compiler.parser.soft.CommonParser._
-import org.alephium.ralph.lsp.access.compiler.parser.soft.ast.SoftAST
+import org.alephium.ralph.lsp.access.compiler.parser.soft.ast.{SoftAST, Token}
 
 private object AssignmentParser {
 
@@ -14,7 +14,7 @@ private object AssignmentParser {
         AssignmentAccessModifierParser.parseOrFail.rep ~
         identifierOrFail ~
         spaceOrFail.? ~
-        TokenParser.EqualOrFail ~
+        TokenParser.parseOrFail(Token.Equal) ~
         spaceOrFail.? ~
         ExpressionParser.parse ~
         Index

compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/BlockParser.scala

@@ -27,11 +27,11 @@ private object BlockParser {
   def clause[Unknown: P](required: Boolean): P[SoftAST.BlockClause] =
     P {
       Index ~
-        TokenParser.OpenCurly(required) ~
+        TokenParser.parse(required, Token.OpenCurly) ~
         spaceOrFail.? ~
         body(Some(Token.CloseCurly.lexeme)) ~
         spaceOrFail.? ~
-        TokenParser.CloseCurly ~
+        TokenParser.parse(Token.CloseCurly) ~
         Index
     } map {
       case (from, openCurly, preBodySpace, body, postBodySpace, closeCurly, to) =>

compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/CodeParser.scala

@@ -0,0 +1,28 @@
+package org.alephium.ralph.lsp.access.compiler.parser.soft
+
+import fastparse._
+import fastparse.NoWhitespace.noWhitespaceImplicit
+import org.alephium.ralph.lsp.access.compiler.message.SourceIndexExtra.range
+import org.alephium.ralph.lsp.access.compiler.parser.soft.ast.{SoftAST, Token}
+
+object CodeParser {
+
+  def parseOrFail[Unknown: P, T <: Token](token: T): P[SoftAST.CodeToken[T]] =
+    P(Index ~ token.lexeme ~ Index) map {
+      case (from, to) =>
+        SoftAST.CodeToken(
+          index = range(from, to),
+          token = token
+        )
+    }
+
+  def parseOrFail[Unknown: P](parser: => P[String]): P[SoftAST.CodeString] =
+    P(Index ~ parser ~ Index) map {
+      case (from, code, to) =>
+        SoftAST.CodeString(
+          index = range(from, to),
+          text = code
+        )
+    }
+
+}

compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/CommentParser.scala

@@ -20,7 +20,7 @@ import fastparse._
 import fastparse.NoWhitespace.noWhitespaceImplicit
 import org.alephium.ralph.lsp.access.compiler.message.SourceIndexExtra.range
 import org.alephium.ralph.lsp.access.compiler.parser.soft.CommonParser._
-import org.alephium.ralph.lsp.access.compiler.parser.soft.ast.SoftAST
+import org.alephium.ralph.lsp.access.compiler.parser.soft.ast.{SoftAST, Token}
 
 private object CommentParser {
 
@@ -63,7 +63,7 @@ private object CommentParser {
   private def one[Unknown: P]: P[SoftAST.Comment] =
     P {
       Index ~
-        TokenParser.DoubleForwardSlashOrFail ~
+        TokenParser.parseOrFailUndocumented(Token.DoubleForwardSlash) ~
         spaceOrFail.? ~
         text.? ~
         spaceOrFail.? ~

compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/CommonParser.scala

@@ -38,10 +38,10 @@ private object CommonParser {
         SoftAST.Space(text)
     }
 
-  def text[Unknown: P]: P[SoftAST.Code] =
+  def text[Unknown: P]: P[SoftAST.CodeString] =
     P(Index ~ CharsWhileNot(Token.Newline.lexeme).! ~ Index) map {
       case (from, text, to) =>
-        SoftAST.Code(
+        SoftAST.CodeString(
           text = text,
           index = range(from, to)
         )
@@ -107,10 +107,10 @@ private object CommonParser {
         )
     }
 
-  def toCodeOrFail[Unknown: P](parser: => P[String]): P[SoftAST.Code] =
+  def toCodeOrFail[Unknown: P](parser: => P[String]): P[SoftAST.CodeString] =
     P(Index ~ parser ~ Index) map {
       case (from, code, to) =>
-        SoftAST.Code(
+        SoftAST.CodeString(
           text = code,
           index = range(from, to)
         )

compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/DataTemplateParser.scala

@@ -4,14 +4,14 @@ import fastparse._
 import fastparse.NoWhitespace.noWhitespaceImplicit
 import org.alephium.ralph.lsp.access.compiler.message.SourceIndexExtra.range
 import org.alephium.ralph.lsp.access.compiler.parser.soft.CommonParser._
-import org.alephium.ralph.lsp.access.compiler.parser.soft.ast.SoftAST
+import org.alephium.ralph.lsp.access.compiler.parser.soft.ast.{SoftAST, Token}
 
 private object DataTemplateParser {
 
   def parseOrFail[Unknown: P]: P[SoftAST.DataTemplate] =
     P {
       Index ~
-        (TokenParser.StructOrFail | TokenParser.EnumOrFail | TokenParser.EventOrFail) ~
+        (TokenParser.parseOrFail(Token.Struct) | TokenParser.parseOrFail(Token.Enum) | TokenParser.parseOrFail(Token.Event)) ~
         space ~
         identifier ~
         spaceOrFail.? ~

compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/ForLoopParser.scala

@@ -4,28 +4,28 @@ import fastparse._
 import fastparse.NoWhitespace.noWhitespaceImplicit
 import org.alephium.ralph.lsp.access.compiler.message.SourceIndexExtra.range
 import org.alephium.ralph.lsp.access.compiler.parser.soft.CommonParser._
-import org.alephium.ralph.lsp.access.compiler.parser.soft.ast.SoftAST
+import org.alephium.ralph.lsp.access.compiler.parser.soft.ast.{SoftAST, Token}
 
 private object ForLoopParser {
 
   def parseOrFail[Unknown: P]: P[SoftAST.ForStatement] =
     P {
       Index ~
-        TokenParser.ForOrFail ~
+        TokenParser.parseOrFail(Token.For) ~
         spaceOrFail.? ~
-        TokenParser.OpenParen ~
+        TokenParser.parse(Token.OpenParen) ~
         spaceOrFail.? ~
         ExpressionParser.parse ~
         spaceOrFail.? ~
-        TokenParser.Semicolon ~
+        TokenParser.parse(Token.Semicolon) ~
         spaceOrFail.? ~
         ExpressionParser.parse ~
         spaceOrFail.? ~
-        TokenParser.Semicolon ~
+        TokenParser.parse(Token.Semicolon) ~
         spaceOrFail.? ~
         ExpressionParser.parse ~
         spaceOrFail.? ~
-        TokenParser.CloseParen ~
+        TokenParser.parse(Token.CloseParen) ~
         spaceOrFail.? ~
         BlockParser.clause(required = true) ~
         Index

compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/FunctionParser.scala

@@ -20,7 +20,7 @@ import fastparse._
 import fastparse.NoWhitespace.noWhitespaceImplicit
 import org.alephium.ralph.lsp.access.compiler.message.SourceIndexExtra.range
 import org.alephium.ralph.lsp.access.compiler.parser.soft.CommonParser._
-import org.alephium.ralph.lsp.access.compiler.parser.soft.ast.SoftAST
+import org.alephium.ralph.lsp.access.compiler.parser.soft.ast.{SoftAST, Token}
 
 private object FunctionParser {
 
@@ -36,7 +36,7 @@ private object FunctionParser {
         AnnotationParser.parseOrFail.rep ~
         spaceOrFail.? ~
         AccessModifierParser.parseOrFail.? ~
-        TokenParser.FnOrFail ~
+        TokenParser.parseOrFail(Token.Fn) ~
         space ~
         signature ~
         spaceOrFail.? ~
@@ -83,7 +83,7 @@ private object FunctionParser {
    *         type or an error indicating that the return type is expected.
    */
   private def returnSignature[Unknown: P]: P[SoftAST.FunctionReturnAST] =
-    P(Index ~ (TokenParser.ForwardArrow ~ spaceOrFail.? ~ TypeParser.parse).? ~ Index) map {
+    P(Index ~ (TokenParser.parse(Token.ForwardArrow) ~ spaceOrFail.? ~ TypeParser.parse).? ~ Index) map {
       case (from, Some((forwardArrow, space, tpe)), to) =>
         SoftAST.FunctionReturn(
           index = range(from, to),

compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/MethodCallParser.scala

@@ -4,7 +4,7 @@ import fastparse._
 import fastparse.NoWhitespace.noWhitespaceImplicit
 import org.alephium.ralph.lsp.access.compiler.message.SourceIndexExtra.range
 import org.alephium.ralph.lsp.access.compiler.parser.soft.CommonParser._
-import org.alephium.ralph.lsp.access.compiler.parser.soft.ast.SoftAST
+import org.alephium.ralph.lsp.access.compiler.parser.soft.ast.{SoftAST, Token}
 
 case object MethodCallParser {
 
@@ -28,7 +28,7 @@ case object MethodCallParser {
   private def dotCall[Unknown: P]: P[SoftAST.DotCall] =
     P {
       Index ~
-        TokenParser.DotOrFail ~
+        TokenParser.parseOrFail(Token.Dot) ~
         spaceOrFail.? ~
         ReferenceCallParser.parse ~
         Index

compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/ReturnStatementParser.scala

@@ -4,14 +4,14 @@ import fastparse._
 import fastparse.NoWhitespace.noWhitespaceImplicit
 import org.alephium.ralph.lsp.access.compiler.message.SourceIndexExtra.range
 import org.alephium.ralph.lsp.access.compiler.parser.soft.CommonParser._
-import org.alephium.ralph.lsp.access.compiler.parser.soft.ast.SoftAST
+import org.alephium.ralph.lsp.access.compiler.parser.soft.ast.{SoftAST, Token}
 
 private object ReturnStatementParser {
 
   def parseOrFail[Unknown: P]: P[SoftAST.ReturnStatement] =
     P {
       Index ~
-        TokenParser.ReturnOrFail ~
+        TokenParser.parseOrFail(Token.Return) ~
         space ~
         ExpressionParser.parse ~
         Index

compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/TemplateParser.scala

@@ -4,14 +4,14 @@ import fastparse._
 import fastparse.NoWhitespace.noWhitespaceImplicit
 import org.alephium.ralph.lsp.access.compiler.message.SourceIndexExtra.range
 import org.alephium.ralph.lsp.access.compiler.parser.soft.CommonParser._
-import org.alephium.ralph.lsp.access.compiler.parser.soft.ast.SoftAST
+import org.alephium.ralph.lsp.access.compiler.parser.soft.ast.{SoftAST, Token}
 
 private object TemplateParser {
 
   def parseOrFail[Unknown: P]: P[SoftAST.Template] =
     P {
       Index ~
-        (TokenParser.ContractOrFail | TokenParser.TxScriptOrFail) ~
+        (TokenParser.parseOrFail(Token.Contract) | TokenParser.parseOrFail(Token.TxScript) | TokenParser.parseOrFail(Token.AssetScript)) ~
         space ~
         identifier ~
         spaceOrFail.? ~
@@ -39,7 +39,7 @@ private object TemplateParser {
   private def inheritance[Unknown: P]: P[SoftAST.TemplateInheritance] =
     P {
       Index ~
-        (TokenParser.ImplementsOrFail | TokenParser.ExtendsOrFail) ~
+        (TokenParser.parseOrFail(Token.Implements) | TokenParser.parseOrFail(Token.Extends)) ~
         space ~
         (ReferenceCallParser.parseOrFail | identifier) ~
         spaceOrFail.? ~