Recognize fewer braces COLONeol's depending on previous token

 - Drop the criterion of no whitespace.
 - Also, refactor the logic so that tokens are converted to COLONeol
   only on demand.

Author: odersky

compiler/src/dotty/tools/dotc/parsing/JavaScanners.scala

@@ -179,7 +179,7 @@ object JavaScanners {
               nextChar()
 
             case ':' =>
-              token = COLON
+              token = COLONop
               nextChar()
 
             case '@' =>

compiler/src/dotty/tools/dotc/parsing/Parsers.scala

@@ -312,7 +312,7 @@ object Parsers {
     def acceptColon(): Int =
       val offset = in.offset
       if in.isColon() then { in.nextToken(); offset }
-      else accept(COLON)
+      else accept(COLONop)
 
     /** semi = nl {nl} | `;'
      *  nl  = `\n' // where allowed
@@ -899,10 +899,11 @@ object Parsers {
     def followingIsSelfType() =
       val lookahead = in.LookaheadScanner(allowIndent = true)
       lookahead.nextToken()
-      lookahead.token == COLON
+      lookahead.token == COLONfollow
       && {
+        lookahead.observeColonEOL(inTemplate = false)
         lookahead.nextToken()
-        canStartTypeTokens(lookahead.token)
+        canStartTypeTokens.contains(lookahead.token)
       }
 
     /** Is current ident a `*`, and is it followed by a `)`, `, )`, `,EOF`? The latter two are not
@@ -1312,7 +1313,8 @@ object Parsers {
 
     def colonAtEOLOpt(): Unit = {
       possibleColonOffset = in.lastOffset
-      if in.token == COLONEOL then in.nextToken()
+      in.observeColonEOL(inTemplate = false)
+      if in.token == COLONeol then in.nextToken()
     }
 
     def argumentStart(): Unit =
@@ -1328,8 +1330,8 @@ object Parsers {
           patch(source, Span(in.offset), "  ")
 
     def possibleTemplateStart(isNew: Boolean = false): Unit =
-      in.observeColonEOL()
-      if in.token == COLONEOL then
+      in.observeColonEOL(inTemplate = true)
+      if in.token == COLONeol then
         if in.lookahead.token == END then in.token = NEWLINE
         else
           in.nextToken()
@@ -2118,7 +2120,7 @@ object Parsers {
             }
           case _ =>
             t
-      case COLON =>
+      case COLONop | COLONfollow =>
         in.nextToken()
         ascription(t, location)
       case _ =>
@@ -2207,7 +2209,7 @@ object Parsers {
         val start = in.offset
         val name = bindingName()
         val t =
-          if (in.token == COLON && location == Location.InBlock) {
+          if ((in.token == COLONop || in.token == COLONfollow) && location == Location.InBlock) {
             report.errorOrMigrationWarning(
               s"This syntax is no longer supported; parameter needs to be enclosed in (...)${rewriteNotice(`future-migration`)}",
               source.atSpan(Span(start, in.lastOffset)),
@@ -2389,7 +2391,7 @@ object Parsers {
                   makeParameter(name.asTermName, typedOpt(), Modifiers(), isBackquoted = isBackquoted(id))
                 }
               case _ => t
-          else if in.fewerBracesEnabled && in.token == COLON && followingIsLambdaAfterColon() then
+          else if in.fewerBracesEnabled && in.token == COLONfollow && followingIsLambdaAfterColon() then
             val app = atSpan(startOffset(t), in.skipToken()) {
               Apply(t, expr(Location.InColonArg) :: Nil)
             }
@@ -2407,7 +2409,6 @@ object Parsers {
       val parents =
         if in.isNestedStart then Nil
         else constrApps(exclude = COMMA)
-      colonAtEOLOpt()
       possibleTemplateStart(isNew = true)
       parents match {
         case parent :: Nil if !in.isNestedStart =>
@@ -2717,7 +2718,7 @@ object Parsers {
      */
     def pattern1(location: Location = Location.InPattern): Tree =
       val p = pattern2()
-      if in.token == COLON then
+      if in.token == COLONop || in.token == COLONfollow then
         in.nextToken()
         ascription(p, location)
       else p
@@ -3827,7 +3828,7 @@ object Parsers {
       val parents =
         if (in.token == EXTENDS) {
           in.nextToken()
-          if (in.token == LBRACE || in.token == COLONEOL) {
+          if (in.token == LBRACE || in.token == COLONeol) {
             report.errorOrMigrationWarning(
               "`extends` must be followed by at least one parent",
               in.sourcePos(), from = `3.0`)
@@ -3948,8 +3949,8 @@ object Parsers {
      */
     def selfType(): ValDef =
       if (in.isIdent || in.token == THIS)
-            && (in.lookahead.token == COLON && followingIsSelfType()
-                || in.lookahead.token == ARROW)
+            && in.lookahead.token == COLONop && followingIsSelfType()
+            || in.lookahead.token == ARROW
       then
         atSpan(in.offset) {
           val selfName =
@@ -3958,11 +3959,11 @@ object Parsers {
               nme.WILDCARD
             else ident()
           val selfTpt =
-            if in.token == COLON then
+            if in.token == COLONfollow then
               in.nextToken()
               infixType()
             else
-              if selfName == nme.WILDCARD then accept(COLON)
+              if selfName == nme.WILDCARD then accept(COLONfollow)
               TypeTree()
           if in.token == ARROW then
             in.token = SELFARROW // suppresses INDENT insertion after `=>`

compiler/src/dotty/tools/dotc/parsing/Scanners.scala

@@ -75,13 +75,13 @@ object Scanners {
     def isNestedStart = token == LBRACE || token == INDENT
     def isNestedEnd = token == RBRACE || token == OUTDENT
 
-    /** Is token a COLON, after having converted COLONEOL to COLON?
+    /** Is token a COLON, after having converted COLONeol to COLON?
      *  The conversion means that indentation is not significant after `:`
      *  anymore. So, warning: this is a side-effecting operation.
      */
     def isColon() =
-      if token == COLONEOL then token = COLON
-      token == COLON
+      if token == COLONeol then token = COLONop
+      token == COLONop || token == COLONfollow
 
     /** Is current token first one after a newline? */
     def isAfterLineEnd: Boolean = lineOffset >= 0
@@ -394,10 +394,11 @@ object Scanners {
      */
     def nextToken(): Unit =
       val lastToken = token
+      val lastName = name
       adjustSepRegions(lastToken)
       getNextToken(lastToken)
       if isAfterLineEnd then handleNewLine(lastToken)
-      postProcessToken(lastToken)
+      postProcessToken(lastToken, lastName)
       printState()
 
     final def printState() =
@@ -428,7 +429,7 @@ object Scanners {
       && {
         // Is current lexeme  assumed to start an expression?
         // This is the case if the lexime is one of the tokens that
-        // starts an expression or it is a COLONEOL. Furthermore, if
+        // starts an expression or it is a COLONeol. Furthermore, if
         // the previous token is in backticks, the lexeme may not be a binary operator.
         // I.e. in
         //
@@ -439,7 +440,7 @@ object Scanners {
         // in backticks and is a binary operator. Hence, `x` is not classified as a
         // leading infix operator.
         def assumeStartsExpr(lexeme: TokenData) =
-          (canStartExprTokens.contains(lexeme.token) || lexeme.token == COLONEOL)
+          (canStartExprTokens.contains(lexeme.token) || lexeme.token == COLONeol)
           && (!lexeme.isOperator || nme.raw.isUnary(lexeme.name))
         val lookahead = LookaheadScanner()
         lookahead.allowLeadingInfixOperators = false
@@ -615,12 +616,11 @@ object Scanners {
             currentRegion match
               case r: Indented =>
                 insert(OUTDENT, offset)
-                if next.token != COLON then
-                  handleNewIndentWidth(r.enclosing, ir =>
-                    errorButContinue(
-                      i"""The start of this line does not match any of the previous indentation widths.
-                          |Indentation width of current line : $nextWidth
-                          |This falls between previous widths: ${ir.width} and $lastWidth"""))
+                handleNewIndentWidth(r.enclosing, ir =>
+                  errorButContinue(
+                    i"""The start of this line does not match any of the previous indentation widths.
+                       |Indentation width of current line : $nextWidth
+                       |This falls between previous widths: ${ir.width} and $lastWidth"""))
               case r =>
                 if skipping then
                   if r.enclosing.isClosedByUndentAt(nextWidth) then
@@ -637,7 +637,7 @@ object Scanners {
             currentRegion.knownWidth = nextWidth
         else if (lastWidth != nextWidth)
           errorButContinue(spaceTabMismatchMsg(lastWidth, nextWidth))
-      if token != OUTDENT || next.token == COLON then
+      if token != OUTDENT then
         handleNewIndentWidth(currentRegion, _.otherIndentWidths += nextWidth)
     end handleNewLine
 
@@ -646,19 +646,22 @@ object Scanners {
          |Previous indent : $lastWidth
          |Latest indent   : $nextWidth"""
 
-    def observeColonEOL(): Unit =
-      if token == COLON then
+    def observeColonEOL(inTemplate: Boolean): Unit =
+      val enabled =
+        if inTemplate then token == COLONop || token == COLONfollow
+        else token == COLONfollow && fewerBracesEnabled
+      if enabled then
         lookAhead()
         val atEOL = isAfterLineEnd || token == EOF
         reset()
-        if atEOL then token = COLONEOL
+        if atEOL then token = COLONeol
 
     def observeIndented(): Unit =
       if indentSyntax && isNewLine then
         val nextWidth = indentWidth(next.offset)
         val lastWidth = currentRegion.indentWidth
         if lastWidth < nextWidth then
-          currentRegion = Indented(nextWidth, COLONEOL, currentRegion)
+          currentRegion = Indented(nextWidth, COLONeol, currentRegion)
           offset = next.offset
           token = INDENT
     end observeIndented
@@ -691,10 +694,10 @@ object Scanners {
       case _ =>
 
     /** - Join CASE + CLASS => CASECLASS, CASE + OBJECT => CASEOBJECT
-     *         SEMI + ELSE => ELSE, COLON + <EOL> => COLONEOL
+     *         SEMI + ELSE => ELSE, COLON following id/)/] => COLONfollow
      *  - Insert missing OUTDENTs at EOF
      */
-    def postProcessToken(lastToken: Token): Unit = {
+    def postProcessToken(lastToken: Token, lastName: SimpleName): Unit = {
       def fuse(tok: Int) = {
         token = tok
         offset = prev.offset
@@ -729,9 +732,10 @@ object Scanners {
                 reset()
         case END =>
           if !isEndMarker then token = IDENTIFIER
-        case COLON =>
-          if fewerBracesEnabled && colonEOLPredecessors.contains(lastToken) && lastOffset == offset then
-            observeColonEOL()
+        case COLONop =>
+          if lastToken == IDENTIFIER && lastName != null && isIdentifierStart(lastName.head)
+              || colonEOLPredecessors.contains(lastToken)
+          then token = COLONfollow
         case RBRACE | RPAREN | RBRACKET =>
           closeIndented()
         case EOF =>
@@ -1188,7 +1192,7 @@ object Scanners {
       isSoftModifier && inModifierPosition()
 
     def isSoftModifierInParamModifierPosition: Boolean =
-      isSoftModifier && lookahead.token != COLON
+      isSoftModifier && lookahead.token != COLONop && lookahead.token != COLONfollow
 
     def isErased: Boolean = isIdent(nme.erased) && erasedEnabled
 
@@ -1527,6 +1531,7 @@ object Scanners {
       case NEWLINE => ";"
       case NEWLINES => ";;"
       case COMMA => ","
+      case COLONfollow | COLONeol => "':'"
       case _ =>
         if debugTokenStream then showTokenDetailed(token) else showToken(token)
     }

compiler/src/dotty/tools/dotc/parsing/Tokens.scala

@@ -16,13 +16,6 @@ abstract class TokensCommon {
 
   def tokenRange(lo: Int, hi: Int): TokenSet = BitSet(lo to hi: _*)
 
-  def showTokenDetailed(token: Int): String = debugString(token)
-
-  def showToken(token: Int): String = {
-    val str = tokenString(token)
-    if (isKeyword(token)) s"'$str'" else str
-  }
-
   val tokenString, debugString: Array[String] = new Array[String](maxToken + 1)
 
   def enter(token: Int, str: String, debugStr: String = ""): Unit = {
@@ -107,7 +100,7 @@ abstract class TokensCommon {
 
   /** special keywords */
   //inline val USCORE = 73;           enter(USCORE, "_")
-  inline val COLON = 74;            enter(COLON, ":")
+  inline val COLONop = 74;          enter(COLONop, ":")  // a stand-alone `:`, see also COLONfollow
   inline val EQUALS = 75;           enter(EQUALS, "=")
   //inline val LARROW = 76;           enter(LARROW, "<-")
   //inline val ARROW = 77;            enter(ARROW, "=>")
@@ -204,8 +197,11 @@ object Tokens extends TokensCommon {
 
   inline val QUOTE = 87;            enter(QUOTE, "'")
 
-  inline val COLONEOL = 88;         enter(COLONEOL, ":", ": at eol")
-  inline val SELFARROW = 89;        enter(SELFARROW, "=>") // reclassified ARROW following self-type
+  inline val COLONfollow = 88;      enter(COLONfollow, ":")
+    // A `:` following an alphanumeric identifier or one of the tokens in colonEOLPredecessors
+  inline val COLONeol = 89;         enter(COLONeol, ":", ": at eol")
+    // A `:` recognized as starting an indentation block
+  inline val SELFARROW = 90;        enter(SELFARROW, "=>") // reclassified ARROW following self-type
 
   /** XML mode */
   inline val XMLSTART = 99;         enter(XMLSTART, "$XMLSTART$<") // TODO: deprecate
@@ -276,7 +272,7 @@ object Tokens extends TokensCommon {
   final val closingRegionTokens = BitSet(RBRACE, RPAREN, RBRACKET, CASE) | statCtdTokens
 
   final val canStartIndentTokens: BitSet =
-    statCtdTokens | BitSet(COLONEOL, WITH, EQUALS, ARROW, CTXARROW, LARROW, WHILE, TRY, FOR, IF, THROW, RETURN)
+    statCtdTokens | BitSet(COLONeol, WITH, EQUALS, ARROW, CTXARROW, LARROW, WHILE, TRY, FOR, IF, THROW, RETURN)
 
   /** Faced with the choice between a type and a formal parameter, the following
    *  tokens determine it's a formal parameter.
@@ -287,9 +283,16 @@ object Tokens extends TokensCommon {
 
   final val endMarkerTokens = identifierTokens | BitSet(IF, WHILE, FOR, MATCH, TRY, NEW, THROW, GIVEN, VAL, THIS)
 
-  final val colonEOLPredecessors = identifierTokens | BitSet(RPAREN, RBRACKET)
+  final val colonEOLPredecessors = BitSet(RPAREN, RBRACKET, BACKQUOTED_IDENT, THIS, SUPER, QUOTEID, STRINGLIT, NEW)
 
   final val closingParens = BitSet(RPAREN, RBRACKET, RBRACE)
 
   final val softModifierNames = Set(nme.inline, nme.opaque, nme.open, nme.transparent, nme.infix)
+
+  def showTokenDetailed(token: Int): String = debugString(token)
+
+  def showToken(token: Int): String = {
+    val str = tokenString(token)
+    if isKeyword(token) || token == COLONfollow || token == COLONeol then s"'$str'" else str
+  }
 }