Revert "Reduce reliance on 'sweet.js' tokenizer algorithm"

This reverts commits effe659 and a17399c

The new approach had at least as many problems as the old, so I'm
going to back away from it again.

src/expression.js

@@ -297,9 +297,7 @@ pp.parseSubscripts = function(base, startPos, startLoc, noCalls) {
 // or `{}`.
 
 pp.parseExprAtom = function(refDestructuringErrors) {
-  this.turnSlashIntoRegexp()
   let node, canBeArrow = this.potentialArrowAt === this.start
-
   switch (this.type) {
   case tt._super:
     if (!this.inFunction)
@@ -529,21 +527,17 @@ pp.parseTemplateElement = function({isTagged}) {
 
 pp.parseTemplate = function({isTagged = false} = {}) {
   let node = this.startNode()
-  this.inTemplate = true
   this.next()
   node.expressions = []
   let curElt = this.parseTemplateElement({isTagged})
   node.quasis = [curElt]
   while (!curElt.tail) {
     if (this.type === tt.eof) this.raise(this.pos, "Unterminated template literal")
-    this.inTemplate = false
     this.expect(tt.dollarBraceL)
     node.expressions.push(this.parseExpression())
-    this.inTemplate = true
     this.expect(tt.braceR)
     node.quasis.push(curElt = this.parseTemplateElement({isTagged}))
   }
-  this.inTemplate = false
   this.next()
   return this.finishNode(node, "TemplateLiteral")
 }
@@ -847,9 +841,22 @@ pp.checkUnreserved = function({start, end, name}) {
 pp.parseIdent = function(liberal, isBinding) {
   let node = this.startNode()
   if (liberal && this.options.allowReserved === "never") liberal = false
-  if (this.type === tt.name) node.name = this.value
-  else if (this.type.keyword) node.name = this.type.keyword
-  else this.unexpected()
+  if (this.type === tt.name) {
+    node.name = this.value
+  } else if (this.type.keyword) {
+    node.name = this.type.keyword
+
+    // To fix https://github.com/acornjs/acorn/issues/575
+    // `class` and `function` keywords push new context into this.context.
+    // But there is no chance to pop the context if the keyword is consumed as an identifier such as a property name.
+    // If the previous token is a dot, this does not apply because the context-managing code already ignored the keyword
+    if ((node.name === "class" || node.name === "function") &&
+        (this.lastTokEnd !== this.lastTokStart + 1 || this.input.charCodeAt(this.lastTokStart) !== 46)) {
+      this.context.pop()
+    }
+  } else {
+    this.unexpected()
+  }
   this.next()
   this.finishNode(node, "Identifier")
   if (!liberal) this.checkUnreserved(node)
@@ -863,7 +870,6 @@ pp.parseYield = function() {
 
   let node = this.startNode()
   this.next()
-  this.turnSlashIntoRegexp()
   if (this.type === tt.semi || this.canInsertSemicolon() || (this.type !== tt.star && !this.type.startsExpr)) {
     node.delegate = false
     node.argument = null

src/loose/expression.js

@@ -340,16 +340,13 @@ lp.parseTemplateElement = function() {
 
 lp.parseTemplate = function() {
   let node = this.startNode()
-  this.toks.inTemplate = true
   this.next()
   node.expressions = []
   let curElt = this.parseTemplateElement()
   node.quasis = [curElt]
   while (!curElt.tail) {
-    this.toks.inTemplate = false
     this.next()
     node.expressions.push(this.parseExpression())
-    this.toks.inTemplate = true
     if (this.expect(tt.braceR)) {
       curElt = this.parseTemplateElement()
     } else {
@@ -360,7 +357,6 @@ lp.parseTemplate = function() {
     }
     node.quasis.push(curElt)
   }
-  this.toks.inTemplate = false
   this.expect(tt.backQuote)
   return this.finishNode(node, "TemplateLiteral")
 }

src/loose/tokenize.js

@@ -1,4 +1,4 @@
-import {tokTypes as tt, isNewLine, SourceLocation, getLineInfo, lineBreakG} from "../index"
+import {tokTypes as tt, Token, isNewLine, SourceLocation, getLineInfo, lineBreakG} from "../index"
 import {LooseParser} from "./state"
 
 const lp = LooseParser.prototype
@@ -26,15 +26,14 @@ lp.next = function() {
 lp.readToken = function() {
   for (;;) {
     try {
-      let tok = this.toks.getToken()
-      if (tok.type === tt.dot &&
-          this.toks.input.substr(tok.end, 1) === "." &&
+      this.toks.next()
+      if (this.toks.type === tt.dot &&
+          this.input.substr(this.toks.end, 1) === "." &&
           this.options.ecmaVersion >= 6) {
         this.toks.end++
-        tok.end++
-        tok.type = tt.ellipsis
+        this.toks.type = tt.ellipsis
       }
-      return tok
+      return new Token(this.toks)
     } catch (e) {
       if (!(e instanceof SyntaxError)) throw e
 

src/state.js

@@ -61,8 +61,7 @@ export class Parser {
     // context to predict whether a regular expression is allowed in a
     // given position.
     this.context = this.initialContext()
-    this.inTemplate = false
-    this.exprAllowed = false
+    this.exprAllowed = true
 
     // Figure out if it's a module code.
     this.inModule = options.sourceType === "module"
@@ -115,8 +114,6 @@ export class Parser {
   }
 
   static tokenizer(input, options) {
-    let parser = new this(options, input)
-    parser.exprAllowed = true
-    return parser
+    return new this(options, input)
   }
 }

src/tokencontext.js

@@ -7,9 +7,11 @@ import {types as tt} from "./tokentype"
 import {lineBreak} from "./whitespace"
 
 export class TokContext {
-  constructor(token, isExpr, generator) {
+  constructor(token, isExpr, preserveSpace, override, generator) {
     this.token = token
     this.isExpr = !!isExpr
+    this.preserveSpace = !!preserveSpace
+    this.override = override
     this.generator = !!generator
   }
 }
@@ -20,11 +22,11 @@ export const types = {
   b_tmpl: new TokContext("${", false),
   p_stat: new TokContext("(", false),
   p_expr: new TokContext("(", true),
-  q_tmpl: new TokContext("`", true),
+  q_tmpl: new TokContext("`", true, true, p => p.tryReadTemplateToken()),
   f_stat: new TokContext("function", false),
   f_expr: new TokContext("function", true),
-  f_expr_gen: new TokContext("function", true, true),
-  f_gen: new TokContext("function", false, true)
+  f_expr_gen: new TokContext("function", true, false, null, true),
+  f_gen: new TokContext("function", false, false, null, true)
 }
 
 const pp = Parser.prototype
@@ -73,10 +75,6 @@ pp.updateContext = function(prevType) {
     this.exprAllowed = type.beforeExpr
 }
 
-pp.curContext = function() {
-  return this.context[this.context.length - 1]
-}
-
 // Token-specific context update code
 
 tt.parenR.updateContext = tt.braceR.updateContext = function() {

src/tokenize.js

@@ -40,19 +40,10 @@ pp.next = function() {
 }
 
 pp.getToken = function() {
-  let prevType = this.type
   this.next()
-  this.updateContext(prevType)
   return new Token(this)
 }
 
-pp.turnSlashIntoRegexp = function() {
-  if (this.type === tt.slash || this.type === tt.assign && this.value === "/=") {
-    this.pos = this.start + 1
-    this.readRegexp()
-  }
-}
-
 // If we're in an ES6 environment, make parsers iterable
 if (typeof Symbol !== "undefined")
   pp[Symbol.iterator] = function() {
@@ -67,17 +58,25 @@ if (typeof Symbol !== "undefined")
     }
   }
 
+// Toggle strict mode. Re-reads the next number or string to please
+// pedantic tests (`"use strict"; 010;` should fail).
+
+pp.curContext = function() {
+  return this.context[this.context.length - 1]
+}
+
 // Read a single token, updating the parser object's token-related
 // properties.
 
 pp.nextToken = function() {
-  if (!this.inTemplate) this.skipSpace()
+  let curContext = this.curContext()
+  if (!curContext || !curContext.preserveSpace) this.skipSpace()
 
   this.start = this.pos
   if (this.options.locations) this.startLoc = this.curPosition()
   if (this.pos >= this.input.length) return this.finishToken(tt.eof)
 
-  if (this.inTemplate) this.tryReadTemplateToken()
+  if (curContext.override) return curContext.override(this)
   else this.readToken(this.fullCharCodeAtPos())
 }
 
@@ -178,8 +177,11 @@ pp.skipSpace = function() {
 pp.finishToken = function(type, val) {
   this.end = this.pos
   if (this.options.locations) this.endLoc = this.curPosition()
+  let prevType = this.type
   this.type = type
   this.value = val
+
+  this.updateContext(prevType)
 }
 
 // ### Token reading