Add support for invalid escapes in tagged templates (babel#274)

Per the stage-3 TC39 proposal: https://github.com/tc39/proposal-template-literal-revision
wcjohnson · Mar 21, 2017 · fab343e · fab343e
1 parent 0811438
commit fab343e
Show file tree

Hide file tree

Showing 290 changed files with 11,491 additions and 38 deletions.
diff --git a/README.md b/README.md
@@ -131,3 +131,4 @@ require("babylon").parse("code", {
  - `functionBind`
  - `functionSent`
  - `dynamicImport`
+ - `templateInvalidEscapes`
diff --git a/ast/spec.md b/ast/spec.md
@@ -959,7 +959,7 @@ interface TemplateElement <: Node {
   type: "TemplateElement";
   tail: boolean;
   value: {
-    cooked: string;
+    cooked: string | null;
     raw: string;
   };
 }

diff --git a/src/parser/expression.js b/src/parser/expression.js
@@ -317,7 +317,7 @@ pp.parseSubscripts = function (base, startPos, startLoc, noCalls) {
     } else if (this.match(tt.backQuote)) {
       const node = this.startNodeAt(startPos, startLoc);
       node.tag = base;
-      node.quasi = this.parseTemplate();
+      node.quasi = this.parseTemplate(true);
       base = this.finishNode(node, "TaggedTemplateExpression");
     } else {
       return base;
@@ -506,7 +506,7 @@ pp.parseExprAtom = function (refShorthandDefaultPos) {
       return this.parseNew();
 
     case tt.backQuote:
-      return this.parseTemplate();
+      return this.parseTemplate(false);
 
     case tt.doubleColon:
       node = this.startNode();
@@ -685,8 +685,15 @@ pp.parseNew = function () {
 
 // Parse template expression.
 
-pp.parseTemplateElement = function () {
+pp.parseTemplateElement = function (isTagged) {
   const elem = this.startNode();
+  if (this.state.value === null) {
+    if (!isTagged || !this.hasPlugin("templateInvalidEscapes")) {
+      this.raise(this.state.invalidTemplateEscapePosition, "Invalid escape sequence in template");
+    } else {
+      this.state.invalidTemplateEscapePosition = null;
+    }
+  }
   elem.value = {
     raw: this.input.slice(this.state.start, this.state.end).replace(/\r\n?/g, "\n"),
     cooked: this.state.value
@@ -696,17 +703,17 @@ pp.parseTemplateElement = function () {
   return this.finishNode(elem, "TemplateElement");
 };
 
-pp.parseTemplate = function () {
+pp.parseTemplate = function (isTagged) {
   const node = this.startNode();
   this.next();
   node.expressions = [];
-  let curElt = this.parseTemplateElement();
+  let curElt = this.parseTemplateElement(isTagged);
   node.quasis = [curElt];
   while (!curElt.tail) {
     this.expect(tt.dollarBraceL);
     node.expressions.push(this.parseExpression());
     this.expect(tt.braceR);
-    node.quasis.push(curElt = this.parseTemplateElement());
+    node.quasis.push(curElt = this.parseTemplateElement(isTagged));
   }
   this.next();
   return this.finishNode(node, "TemplateLiteral");

diff --git a/src/tokenizer/index.js b/src/tokenizer/index.js
@@ -599,17 +599,26 @@ export default class Tokenizer {
 
   // Read a string value, interpreting backslash-escapes.
 
-  readCodePoint() {
+  readCodePoint(throwOnInvalid) {
     const ch = this.input.charCodeAt(this.state.pos);
     let code;
 
-    if (ch === 123) {
+    if (ch === 123) { // '{'
       const codePos = ++this.state.pos;
-      code = this.readHexChar(this.input.indexOf("}", this.state.pos) - this.state.pos);
+      code = this.readHexChar(this.input.indexOf("}", this.state.pos) - this.state.pos, throwOnInvalid);
       ++this.state.pos;
-      if (code > 0x10FFFF) this.raise(codePos, "Code point out of bounds");
+      if (code === null) {
+        --this.state.invalidTemplateEscapePosition; // to point to the '\'' instead of the 'u'
+      } else if (code > 0x10FFFF) {
+        if (throwOnInvalid) {
+          this.raise(codePos, "Code point out of bounds");
+        } else {
+          this.state.invalidTemplateEscapePosition = codePos - 2;
+          return null;
+        }
+      }
     } else {
-      code = this.readHexChar(4);
+      code = this.readHexChar(4, throwOnInvalid);
     }
     return code;
   }
@@ -636,7 +645,7 @@ export default class Tokenizer {
   // Reads template string tokens.
 
   readTmplToken() {
-    let out = "", chunkStart = this.state.pos;
+    let out = "", chunkStart = this.state.pos, containsInvalid = false;
     for (;;) {
       if (this.state.pos >= this.input.length) this.raise(this.state.start, "Unterminated template");
       const ch = this.input.charCodeAt(this.state.pos);
@@ -651,11 +660,16 @@ export default class Tokenizer {
           }
         }
         out += this.input.slice(chunkStart, this.state.pos);
-        return this.finishToken(tt.template, out);
+        return this.finishToken(tt.template, containsInvalid ? null : out);
       }
       if (ch === 92) { // '\'
         out += this.input.slice(chunkStart, this.state.pos);
-        out += this.readEscapedChar(true);
+        const escaped = this.readEscapedChar(true);
+        if (escaped === null) {
+          containsInvalid = true;
+        } else {
+          out += escaped;
+        }
         chunkStart = this.state.pos;
       } else if (isNewLine(ch)) {
         out += this.input.slice(chunkStart, this.state.pos);
@@ -682,13 +696,20 @@ export default class Tokenizer {
   // Used to read escaped characters
 
   readEscapedChar(inTemplate) {
+    const throwOnInvalid = !inTemplate;
     const ch = this.input.charCodeAt(++this.state.pos);
     ++this.state.pos;
     switch (ch) {
       case 110: return "\n"; // 'n' -> '\n'
       case 114: return "\r"; // 'r' -> '\r'
-      case 120: return String.fromCharCode(this.readHexChar(2)); // 'x'
-      case 117: return codePointToString(this.readCodePoint()); // 'u'
+      case 120: { // 'x'
+        const code = this.readHexChar(2, throwOnInvalid);
+        return code === null ? null : String.fromCharCode(code);
+      }
+      case 117: { // 'u'
+        const code = this.readCodePoint(throwOnInvalid);
+        return code === null ? null : codePointToString(code);
+      }
       case 116: return "\t"; // 't' -> '\t'
       case 98: return "\b"; // 'b' -> '\b'
       case 118: return "\u000b"; // 'v' -> '\u000b'
@@ -700,19 +721,24 @@ export default class Tokenizer {
         return "";
       default:
         if (ch >= 48 && ch <= 55) {
+          const codePos = this.state.pos - 1;
           let octalStr = this.input.substr(this.state.pos - 1, 3).match(/^[0-7]+/)[0];
           let octal = parseInt(octalStr, 8);
           if (octal > 255) {
             octalStr = octalStr.slice(0, -1);
             octal = parseInt(octalStr, 8);
           }
           if (octal > 0) {
-            if (!this.state.containsOctal) {
+            if (inTemplate) {
+              this.state.invalidTemplateEscapePosition = codePos;
+              return null;
+            } else if (this.state.strict) {
+              this.raise(codePos, "Octal literal in strict mode");
+            } else if (!this.state.containsOctal) {
+              // These properties are only used to throw an error for an octal which occurs
+              // in a directive which occurs prior to a "use strict" directive.
               this.state.containsOctal = true;
-              this.state.octalPosition = this.state.pos - 2;
-            }
-            if (this.state.strict || inTemplate) {
-              this.raise(this.state.pos - 2, "Octal literal in strict mode");
+              this.state.octalPosition = codePos;
             }
           }
           this.state.pos += octalStr.length - 1;
@@ -722,12 +748,19 @@ export default class Tokenizer {
     }
   }
 
-  // Used to read character escape sequences ('\x', '\u', '\U').
+  // Used to read character escape sequences ('\x', '\u').
 
-  readHexChar(len) {
+  readHexChar(len, throwOnInvalid) {
     const codePos = this.state.pos;
     const n = this.readInt(16, len);
-    if (n === null) this.raise(codePos, "Bad character escape sequence");
+    if (n === null) {
+      if (throwOnInvalid) {
+        this.raise(codePos, "Bad character escape sequence");
+      } else {
+        this.state.pos = codePos - 1;
+        this.state.invalidTemplateEscapePosition = codePos - 1;
+      }
+    }
     return n;
   }
 
@@ -755,7 +788,7 @@ export default class Tokenizer {
         }
 
         ++this.state.pos;
-        const esc = this.readCodePoint();
+        const esc = this.readCodePoint(true);
         if (!(first ? isIdentifierStart : isIdentifierChar)(esc, true)) {
           this.raise(escStart, "Invalid Unicode escape");
         }

diff --git a/src/tokenizer/state.js b/src/tokenizer/state.js
@@ -50,6 +50,8 @@ export default class State {
     this.containsEsc = this.containsOctal = false;
     this.octalPosition = null;
 
+    this.invalidTemplateEscapePosition = null;
+
     this.exportedIdentifiers = [];
 
     return this;

diff --git a/test/fixtures/core/uncategorised/499/options.json b/test/fixtures/core/uncategorised/499/options.json
@@ -1,3 +1,3 @@
 {
-  "throws": "Octal literal in strict mode (1:34)"
+  "throws": "Octal literal in strict mode (1:35)"
 }
diff --git a/test/fixtures/core/uncategorised/501/options.json b/test/fixtures/core/uncategorised/501/options.json
@@ -1,3 +1,3 @@
 {
-  "throws": "Octal literal in strict mode (1:37)"
+  "throws": "Octal literal in strict mode (1:38)"
 }
diff --git a/test/fixtures/core/uncategorised/503/options.json b/test/fixtures/core/uncategorised/503/options.json
@@ -1,3 +1,3 @@
 {
-  "throws": "Octal literal in strict mode (1:68)"
+  "throws": "Octal literal in strict mode (1:69)"
 }
diff --git a/test/fixtures/es2015/uncategorised/290/options.json b/test/fixtures/es2015/uncategorised/290/options.json
@@ -1,3 +1,3 @@
 {
-  "throws": "Octal literal in strict mode (1:22)"
+  "throws": "Invalid escape sequence in template (1:23)"
 }
diff --git a/test/fixtures/es2015/uncategorised/339/options.json b/test/fixtures/es2015/uncategorised/339/options.json
@@ -1,3 +1,3 @@
 {
-  "throws": "Octal literal in strict mode (1:1)"
+  "throws": "Invalid escape sequence in template (1:2)"
 }
diff --git a/test/fixtures/esprima/es2015-template-literals/invalid-escape/options.json b/test/fixtures/esprima/es2015-template-literals/invalid-escape/options.json
@@ -1,3 +1,3 @@
 {
-  "throws": "Octal literal in strict mode (1:1)"
+  "throws": "Invalid escape sequence in template (1:2)"
 }
diff --git a/test/fixtures/esprima/invalid-syntax/migrated_0216/options.json b/test/fixtures/esprima/invalid-syntax/migrated_0216/options.json
@@ -1,3 +1,3 @@
 {
-  "throws": "Octal literal in strict mode (1:1)"
+  "throws": "Octal literal in strict mode (1:2)"
 }
diff --git a/test/fixtures/esprima/invalid-syntax/migrated_0217/options.json b/test/fixtures/esprima/invalid-syntax/migrated_0217/options.json
@@ -1,3 +1,3 @@
 {
-  "throws": "Octal literal in strict mode (1:34)"
+  "throws": "Octal literal in strict mode (1:35)"
 }
diff --git a/test/fixtures/esprima/invalid-syntax/migrated_0219/options.json b/test/fixtures/esprima/invalid-syntax/migrated_0219/options.json
@@ -1,3 +1,3 @@
 {
-  "throws": "Octal literal in strict mode (1:37)"
+  "throws": "Octal literal in strict mode (1:38)"
 }
diff --git a/test/fixtures/esprima/invalid-syntax/migrated_0221/options.json b/test/fixtures/esprima/invalid-syntax/migrated_0221/options.json
@@ -1,3 +1,3 @@
 {
-  "throws": "Octal literal in strict mode (1:35)"
+  "throws": "Octal literal in strict mode (1:36)"
 }
diff --git a/test/fixtures/esprima/invalid-syntax/migrated_0222/options.json b/test/fixtures/esprima/invalid-syntax/migrated_0222/options.json
@@ -1,3 +1,3 @@
 {
-  "throws": "Octal literal in strict mode (1:35)"
+  "throws": "Octal literal in strict mode (1:36)"
 }
diff --git a/test/fixtures/esprima/invalid-syntax/migrated_0223/options.json b/test/fixtures/esprima/invalid-syntax/migrated_0223/options.json
@@ -1,3 +1,3 @@
 {
-  "throws": "Octal literal in strict mode (1:68)"
+  "throws": "Octal literal in strict mode (1:69)"
 }
diff --git a/test/fixtures/experimental/template-literal-invalid-escapes-tagged/1/actual.js b/test/fixtures/experimental/template-literal-invalid-escapes-tagged/1/actual.js
@@ -0,0 +1 @@
+sampleTag`\01`