Refactor: create new grammar for doc tags

LiquidDoc is essentially a new language that we are adding.
Creating a new grammar allows us to directly parse and test the contents of the doc tag.
This also allows us to avoid the complexity of including liquid support from within the doc tag grammar.

packages/liquid-html-parser/grammar/liquid-html.ohm

@@ -376,7 +376,11 @@ LiquidStatement <: Liquid {
 
   liquidStatementEnd = newline | end
   delimTag := liquidStatementEnd
-} 
+}
+
+LiquidDoc <: Helpers {
+  Node := (TextNode)*
+}
 
 LiquidHTML <: Liquid {
   Node := yamlFrontmatter? (HtmlNode | liquidNode | TextNode)*

packages/liquid-html-parser/src/grammar.ts

@@ -3,6 +3,7 @@ import ohm from 'ohm-js';
 export const liquidHtmlGrammars = ohm.grammars(require('../grammar/liquid-html.ohm.js'));
 
 export const TextNodeGrammar = liquidHtmlGrammars['Helpers'];
+export const LiquidDocGrammar = liquidHtmlGrammars['LiquidDoc'];
 
 export interface LiquidGrammars {
   Liquid: ohm.Grammar;

packages/liquid-html-parser/src/stage-1-cst.spec.ts

@@ -994,13 +994,15 @@ describe('Unit: Stage 1 (CST)', () => {
           expectPath(cst, '0.blockStartLocEnd').to.equal(0 + '{% doc -%}'.length);
           expectPath(cst, '0.blockEndLocStart').to.equal(testStr.length - '{%- enddoc %}'.length);
           expectPath(cst, '0.blockEndLocEnd').to.equal(testStr.length);
-          expectPath(cst, '0.children').to.deep.equal({
-            locEnd: 35,
-            locStart: 11,
-            source: '{% doc -%} Renders loading-spinner. {%- enddoc %}',
-            type: 'LiquidDocBody',
-            description: 'Renders loading-spinner.',
-          });
+          expectPath(cst, '0.children').to.deep.equal([
+            {
+              locEnd: 25,
+              locStart: 1,
+              source: '{% doc -%} Renders loading-spinner. {%- enddoc %}',
+              type: 'TextNode',
+              value: 'Renders loading-spinner.',
+            },
+          ]);
         }
       });
 

packages/liquid-html-parser/src/stage-1-cst.ts

@@ -34,6 +34,7 @@ import { Parser } from 'prettier';
 import ohm, { Node } from 'ohm-js';
 import { toAST } from 'ohm-js/extras';
 import {
+  LiquidDocGrammar,
   LiquidGrammars,
   TextNodeGrammar,
   placeholderGrammars,
@@ -81,7 +82,6 @@ export enum ConcreteNodeTypes {
   RenderMarkup = 'RenderMarkup',
   PaginateMarkup = 'PaginateMarkup',
   RenderVariableExpression = 'RenderVariableExpression',
-  LiquidDocBody = 'LiquidDocBody',
 }
 
 export const LiquidLiteralValues = {
@@ -504,13 +504,7 @@ function toCST<T>(
   source: string /* the original file */,
   grammars: LiquidGrammars,
   grammar: ohm.Grammar,
-  cstMappings: (
-    | 'HelperMappings'
-    | 'LiquidMappings'
-    | 'LiquidHTMLMappings'
-    | 'LiquidStatement'
-    | 'LiquidDocMappings'
-  )[],
+  cstMappings: ('HelperMappings' | 'LiquidMappings' | 'LiquidHTMLMappings' | 'LiquidStatement')[],
   matchingSource: string = source /* for subtree parsing */,
   offset: number = 0 /* for subtree parsing location offsets */,
 ): T {
@@ -638,11 +632,8 @@ function toCST<T>(
       body: (tokens: Node[]) => tokens[1].sourceString,
       children: (tokens: Node[]) => {
         const contentNode = tokens[1];
-        return toCST(
+        return toLiquidDocAST(
           source,
-          grammars,
-          grammars.Liquid,
-          ['LiquidDocMappings'],
           contentNode.sourceString,
           offset + contentNode.source.startIdx,
         );
@@ -1108,15 +1099,24 @@ function toCST<T>(
   };
 
   const LiquidDocMappings: Mapping = {
-    Node: {
-      type: ConcreteNodeTypes.LiquidDocBody,
-      locStart,
-      locEnd,
-      source,
-      description: (tokens: Node[]) => tokens[0].sourceString,
-    },
+    Node: 0,
+    TextNode: textNode,
   };
 
+  function toLiquidDocAST(source: string, matchingSource: string, offset: number) {
+    const res = LiquidDocGrammar.match(matchingSource, 'Node');
+    if (res.failed()) {
+      throw new LiquidHTMLCSTParsingError(res);
+    }
+
+    const LiquidDocMappings: Mapping = {
+      Node: 0,
+      TextNode: textNode,
+    };
+
+    return toAST(res, LiquidDocMappings);
+  }
+
   const LiquidHTMLMappings: Mapping = {
     Node(frontmatter: Node, nodes: Node) {
       const self = this as any;
@@ -1272,7 +1272,6 @@ function toCST<T>(
     LiquidMappings,
     LiquidHTMLMappings,
     LiquidStatement,
-    LiquidDocMappings,
   };
 
   const selectedMappings = cstMappings.reduce(

packages/liquid-html-parser/src/stage-2-ast.spec.ts

@@ -1225,21 +1225,25 @@ describe('Unit: Stage 2 (AST)', () => {
       expectPath(ast, 'children.0.type').to.eql('LiquidRawTag');
       expectPath(ast, 'children.0.name').to.eql('doc');
       expectPath(ast, 'children.0.body.value').to.eql('');
+      expectPath(ast, 'children.0.body.type').toEqual('RawMarkup');
+      expectPath(ast, 'children.0.body.nodes').toEqual([]);
 
       ast = toLiquidAST(`{% doc -%} single line doc {%- enddoc %}`);
       expectPath(ast, 'children.0.type').to.eql('LiquidRawTag');
       expectPath(ast, 'children.0.name').to.eql('doc');
       expectPath(ast, 'children.0.body.value').to.eql(' single line doc ');
+      expectPath(ast, 'children.0.body.nodes.0.type').toEqual('TextNode');
 
       ast = toLiquidAST(`{% doc -%}
         multi line doc
         multi line doc
         {%- enddoc %}`);
       expectPath(ast, 'children.0.type').to.eql('LiquidRawTag');
       expectPath(ast, 'children.0.name').to.eql('doc');
-      expectPath(ast, 'children.0.body.source').to.eql(
-        '{% doc -%}\n        multi line doc\n        multi line doc\n        {%- enddoc %}',
+      expectPath(ast, 'children.0.body.nodes.0.value').to.eql(
+        `multi line doc\n        multi line doc`,
       );
+      expectPath(ast, 'children.0.body.nodes.0.type').toEqual('TextNode');
     });
 
     it('should parse unclosed tables with assignments', () => {