Add codemod mode for @glimmer/syntax preprocess.

Currently, setting `mode` to `'codemod'` will: * Disable `entity` parsing in simple-html-tokenizer (ensures that parsing + printing is not lossy) * Enable `ignoreStandalone` mode in handlebars parser (ensures that standalone whitespace after a block opening or closing is not stripped). In the future we can do other things to make the codemod world even better, but this is a really good first step (IMHO).
glimmerjs · Apr 30, 2019 · fb564f2 · fb564f2
1 parent 157ffdb
commit fb564f2
Show file tree

Hide file tree

Showing 5 changed files with 201 additions and 210 deletions.
diff --git a/packages/@glimmer/syntax/lib/generation/print.ts b/packages/@glimmer/syntax/lib/generation/print.ts
@@ -7,10 +7,63 @@ function unreachable(): never {
   throw new Error('unreachable');
 }
 
-export default function build(ast: AST.Node): string {
+interface PrinterOptions {
+  entityEncoding: 'transformed' | 'raw';
+}
+
+export default function build(
+  ast: AST.Node,
+  options: PrinterOptions = { entityEncoding: 'transformed' }
+): string {
   if (!ast) {
     return '';
   }
+
+  function buildEach(asts: AST.Node[]): string[] {
+    return asts.map(node => build(node, options));
+  }
+
+  function pathParams(ast: AST.Node): string {
+    let path: string;
+
+    switch (ast.type) {
+      case 'MustacheStatement':
+      case 'SubExpression':
+      case 'ElementModifierStatement':
+      case 'BlockStatement':
+        path = build(ast.path, options);
+        break;
+      case 'PartialStatement':
+        path = build(ast.name, options);
+        break;
+      default:
+        return unreachable();
+    }
+
+    return compactJoin([path, buildEach(ast.params).join(' '), build(ast.hash, options)], ' ');
+  }
+
+  function compactJoin(array: Option<string>[], delimiter?: string): string {
+    return compact(array).join(delimiter || '');
+  }
+
+  function blockParams(block: AST.BlockStatement): Option<string> {
+    const params = block.program.blockParams;
+    if (params.length) {
+      return ` as |${params.join(' ')}|`;
+    }
+
+    return null;
+  }
+
+  function openBlock(block: AST.BlockStatement): string {
+    return ['{{#', pathParams(block), blockParams(block), '}}'].join('');
+  }
+
+  function closeBlock(block: any): string {
+    return ['{{/', build(block.path, options), '}}'].join('');
+  }
+
   const output: string[] = [];
 
   switch (ast.type) {
@@ -58,29 +111,33 @@ export default function build(ast: AST.Node): string {
       if (ast.value.type === 'TextNode') {
         if (ast.value.chars !== '') {
           output.push(ast.name, '=');
-          output.push('"', escapeAttrValue(ast.value.chars), '"');
+          output.push(
+            '"',
+            options.entityEncoding === 'raw' ? ast.value.chars : escapeAttrValue(ast.value.chars),
+            '"'
+          );
         } else {
           output.push(ast.name);
         }
       } else {
         output.push(ast.name, '=');
         // ast.value is mustache or concat
-        output.push(build(ast.value));
+        output.push(build(ast.value, options));
       }
       break;
     case 'ConcatStatement':
       output.push('"');
       ast.parts.forEach((node: AST.TextNode | AST.MustacheStatement) => {
         if (node.type === 'TextNode') {
-          output.push(escapeAttrValue(node.chars));
+          output.push(options.entityEncoding === 'raw' ? node.chars : escapeAttrValue(node.chars));
         } else {
-          output.push(build(node));
+          output.push(build(node, options));
         }
       });
       output.push('"');
       break;
     case 'TextNode':
-      output.push(escapeText(ast.chars));
+      output.push(options.entityEncoding === 'raw' ? ast.chars : escapeText(ast.chars));
       break;
     case 'MustacheStatement':
       {
@@ -118,13 +175,13 @@ export default function build(ast: AST.Node): string {
           lines.push(openBlock(ast));
         }
 
-        lines.push(build(ast.program));
+        lines.push(build(ast.program, options));
 
         if (ast.inverse) {
           if (!ast.inverse.chained) {
             lines.push('{{else}}');
           }
-          lines.push(build(ast.inverse));
+          lines.push(build(ast.inverse, options));
         }
 
         if (!ast.chained) {
@@ -169,15 +226,15 @@ export default function build(ast: AST.Node): string {
         output.push(
           ast.pairs
             .map(pair => {
-              return build(pair);
+              return build(pair, options);
             })
             .join(' ')
         );
       }
       break;
     case 'HashPair':
       {
-        output.push(`${ast.key}=${build(ast.value)}`);
+        output.push(`${ast.key}=${build(ast.value, options)}`);
       }
       break;
   }
@@ -193,48 +250,3 @@ function compact(array: Option<string>[]): string[] {
   });
   return newArray;
 }
-
-function buildEach(asts: AST.Node[]): string[] {
-  return asts.map(build);
-}
-
-function pathParams(ast: AST.Node): string {
-  let path: string;
-
-  switch (ast.type) {
-    case 'MustacheStatement':
-    case 'SubExpression':
-    case 'ElementModifierStatement':
-    case 'BlockStatement':
-      path = build(ast.path);
-      break;
-    case 'PartialStatement':
-      path = build(ast.name);
-      break;
-    default:
-      return unreachable();
-  }
-
-  return compactJoin([path, buildEach(ast.params).join(' '), build(ast.hash)], ' ');
-}
-
-function compactJoin(array: Option<string>[], delimiter?: string): string {
-  return compact(array).join(delimiter || '');
-}
-
-function blockParams(block: AST.BlockStatement): Option<string> {
-  const params = block.program.blockParams;
-  if (params.length) {
-    return ` as |${params.join(' ')}|`;
-  }
-
-  return null;
-}
-
-function openBlock(block: AST.BlockStatement): string {
-  return ['{{#', pathParams(block), blockParams(block), '}}'].join('');
-}
-
-function closeBlock(block: any): string {
-  return ['{{/', build(block.path), '}}'].join('');
-}
diff --git a/packages/@glimmer/syntax/lib/parser.ts b/packages/@glimmer/syntax/lib/parser.ts
@@ -8,8 +8,6 @@ import * as HBS from './types/handlebars-ast';
 import { Option } from '@glimmer/interfaces';
 import { assert, expect } from '@glimmer/util';
 
-const entityParser = new EntityParser(namedCharRefs);
-
 export type Element = AST.Template | AST.Block | AST.ElementNode;
 
 export interface Tag<T extends 'StartTag' | 'EndTag'> {
@@ -39,10 +37,11 @@ export abstract class Parser {
   public currentNode: Option<
     AST.CommentStatement | AST.TextNode | Tag<'StartTag' | 'EndTag'>
   > = null;
-  public tokenizer = new EventedTokenizer(this, entityParser);
+  public tokenizer: EventedTokenizer;
 
-  constructor(source: string) {
+  constructor(source: string, entityParser = new EntityParser(namedCharRefs)) {
     this.source = source.split(/(?:\r\n?|\n)/g);
+    this.tokenizer = new EventedTokenizer(this, entityParser);
   }
 
   abstract Program(node: HBS.Program): HBS.Output<'Program'>;

diff --git a/packages/@glimmer/syntax/lib/parser/tokenizer-event-handlers.ts b/packages/@glimmer/syntax/lib/parser/tokenizer-event-handlers.ts
@@ -12,6 +12,7 @@ import Walker from '../traversal/walker';
 import * as handlebars from 'handlebars';
 import { assign } from '@glimmer/util';
 import { NodeVisitor } from '../traversal/visitor';
+import { EntityParser } from 'simple-html-tokenizer';
 
 export const voidMap: {
   [tagName: string]: boolean;
@@ -344,7 +345,16 @@ export interface PreprocessOptions {
   plugins?: {
     ast?: ASTPluginBuilder[];
   };
-  parseOptions?: object;
+  parseOptions?: Handlebars.ParseOptions;
+
+  /**
+    Useful for specifying a group of options together.
+
+    When `'codemod'` we disable all whitespace control in handlebars
+    (to preserve as much as possible) and we also avoid any
+    escaping/unescaping of HTML entity codes.
+   */
+  mode?: 'codemod' | 'precompile';
 }
 
 export interface Syntax {
@@ -363,10 +373,28 @@ const syntax: Syntax = {
   Walker,
 };
 
-export function preprocess(html: string, options?: PreprocessOptions): AST.Template {
-  const parseOptions = options ? options.parseOptions : {};
-  let ast = typeof html === 'object' ? html : (handlebars.parse(html, parseOptions) as HBS.Program);
-  let program = new TokenizerEventHandlers(html).acceptTemplate(ast);
+export function preprocess(html: string, options: PreprocessOptions = {}): AST.Template {
+  let mode = options.mode || 'precompile';
+
+  let ast: HBS.Program;
+  if (typeof html === 'object') {
+    ast = html;
+  } else {
+    let parseOptions = options.parseOptions || {};
+
+    if (mode === 'codemod') {
+      parseOptions.ignoreStandalone = true;
+    }
+
+    ast = handlebars.parse(html, parseOptions) as HBS.Program;
+  }
+
+  let entityParser = undefined;
+  if (mode === 'codemod') {
+    entityParser = new EntityParser({});
+  }
+
+  let program = new TokenizerEventHandlers(html, entityParser).acceptTemplate(ast);
 
   if (options && options.plugins && options.plugins.ast) {
     for (let i = 0, l = options.plugins.ast.length; i < l; i++) {

diff --git a/packages/@glimmer/syntax/lib/types/nodes.ts b/packages/@glimmer/syntax/lib/types/nodes.ts
@@ -57,6 +57,8 @@ export interface Block extends CommonProgram {
   symbols?: BlockSymbols;
 }
 
+export type EntityEncodingState = 'transformed' | 'raw';
+
 export interface Template extends CommonProgram {
   type: 'Template';
   symbols?: Symbols;