test block quote

markedjs · Apr 20, 2024 · a4980a7 · a4980a7
1 parent c6a98ea
commit a4980a7
Show file tree

Hide file tree

Showing 5 changed files with 83 additions and 24 deletions.
diff --git a/src/Lexer.ts b/src/Lexer.ts
@@ -101,9 +101,9 @@ export class _Lexer {
   /**
    * Lexing
    */
-  blockTokens(src: string, tokens?: Token[]): Token[];
-  blockTokens(src: string, tokens?: TokensList): TokensList;
-  blockTokens(src: string, tokens: Token[] = []) {
+  blockTokens(src: string, tokens?: Token[], lastParagraphClipped?: boolean): Token[];
+  blockTokens(src: string, tokens?: TokensList, lastParagraphClipped?: boolean): TokensList;
+  blockTokens(src: string, tokens: Token[] = [], lastParagraphClipped = false) {
     if (this.options.pedantic) {
       src = src.replace(/\t/g, '    ').replace(/^ +$/gm, '');
     } else {
@@ -115,7 +115,6 @@ export class _Lexer {
     let token: Tokens.Generic | undefined;
     let lastToken;
     let cutSrc;
-    let lastParagraphClipped;
 
     while (src) {
       if (this.options.extensions
@@ -249,7 +248,7 @@ export class _Lexer {
       }
       if (this.state.top && (token = this.tokenizer.paragraph(cutSrc))) {
         lastToken = tokens[tokens.length - 1];
-        if (lastParagraphClipped && lastToken.type === 'paragraph') {
+        if (lastParagraphClipped && lastToken?.type === 'paragraph') {
           lastToken.raw += '\n' + token.raw;
           lastToken.text += '\n' + token.text;
           this.inlineQueue.pop();

diff --git a/src/Tokenizer.ts b/src/Tokenizer.ts
@@ -7,7 +7,7 @@ import {
 } from './helpers.ts';
 import type { Rules } from './rules.ts';
 import type { _Lexer } from './Lexer.ts';
-import type { Links, Tokens } from './Tokens.ts';
+import type { Links, Tokens, Token } from './Tokens.ts';
 import type { MarkedOptions } from './MarkedOptions.ts';
 
 function outputLink(cap: string[], link: Pick<Tokens.Link, 'href' | 'title'>, raw: string, lexer: _Lexer): Tokens.Link | Tokens.Image {
@@ -156,20 +156,83 @@ export class _Tokenizer {
   blockquote(src: string): Tokens.Blockquote | undefined {
     const cap = this.rules.block.blockquote.exec(src);
     if (cap) {
+      const lines = rtrim(cap[0], '\n').split('\n');
+
+      return {
+        type: 'blockquote',
+        ...this.testBlockquote(lines)
+      };
+    }
+  }
+
+  testBlockquote(lines: string[]) {
+    let raw = '';
+    let text = '';
+    const tokens: Token[] = [];
+
+    while (lines.length > 0) {
+      let inBlockquote = false;
+      const currentLines = [];
+
+      while (lines.length > 0) {
+        if (/^ {0,3}>/.test(lines[0])) {
+          currentLines.push(lines.shift());
+          inBlockquote = true;
+        } else if (!inBlockquote) {
+          currentLines.push(lines.shift());
+        } else {
+          break;
+        }
+      }
+
+      const currentRaw = currentLines.join('\n');
+      const currentText = currentRaw
       // precede setext continuation with 4 spaces so it isn't a setext
-      let text = cap[0].replace(/\n {0,3}((?:=+|-+) *)(?=\n|$)/g, '\n    $1');
-      text = rtrim(text.replace(/^ *>[ \t]?/gm, ''), '\n');
+        .replace(/\n {0,3}((?:=+|-+) *)(?=\n|$)/g, '\n    $1')
+        .replace(/^ {0,3}>[ \t]?/gm, '');
+      raw = raw ? `${raw}\n${currentRaw}` : currentRaw;
+      text = text ? `${text}\n${currentText}` : currentText;
       const top = this.lexer.state.top;
       this.lexer.state.top = true;
-      const tokens = this.lexer.blockTokens(text);
+      this.lexer.blockTokens(currentText, tokens, true);
       this.lexer.state.top = top;
-      return {
-        type: 'blockquote',
-        raw: cap[0],
-        tokens,
-        text
-      };
+
+      if (lines.length === 0) {
+        break;
+      }
+
+      const lastToken = tokens[tokens.length - 1];
+
+      if (lastToken?.type === 'code') {
+        break;
+      } else if (lastToken?.type === 'blockquote') {
+        const newLines = lastToken.raw.split('\n').concat(lines);
+
+        tokens[tokens.length - 1] = {
+          type: 'blockquote',
+          ...this.testBlockquote(newLines)
+        };
+
+        raw = raw.substring(0, raw.length - lastToken.raw.length) + tokens[tokens.length - 1].raw;
+        text = text.substring(0, text.length - (lastToken as Tokens.Blockquote).text.length) + (tokens[tokens.length - 1] as Tokens.Blockquote).text;
+        break;
+      } else if (lastToken?.type === 'list') {
+        const newLines = lastToken.raw + '\n' + lines.join('\n');
+
+        tokens[tokens.length - 1] = this.list(newLines)!; // TODO: remove `!`
+
+        raw = raw.substring(0, raw.length - lastToken.raw.length) + tokens[tokens.length - 1].raw;
+        text = text.substring(0, text.length - (lastToken as Tokens.List).raw.length) + (tokens[tokens.length - 1] as Tokens.List).raw;
+        lines = newLines.substring(tokens[tokens.length - 1].raw.length).split('\n');
+        continue;
+      }
     }
+
+    return {
+      raw,
+      tokens,
+      text
+    };
   }
 
   list(src: string): Tokens.List | undefined {

diff --git a/test/specs/commonmark/commonmark.0.31.json b/test/specs/commonmark/commonmark.0.31.json
@@ -1887,17 +1887,15 @@
     "example": 236,
     "start_line": 3838,
     "end_line": 3848,
-    "section": "Block quotes",
-    "shouldFail": true
+    "section": "Block quotes"
   },
   {
     "markdown": "> ```\nfoo\n```\n",
     "html": "<blockquote>\n<pre><code></code></pre>\n</blockquote>\n<p>foo</p>\n<pre><code></code></pre>\n",
     "example": 237,
     "start_line": 3851,
     "end_line": 3861,
-    "section": "Block quotes",
-    "shouldFail": true
+    "section": "Block quotes"
   },
   {
     "markdown": "> foo\n    - bar\n",

diff --git a/test/specs/gfm/commonmark.0.31.json b/test/specs/gfm/commonmark.0.31.json
@@ -1887,17 +1887,15 @@
     "example": 236,
     "start_line": 3838,
     "end_line": 3848,
-    "section": "Block quotes",
-    "shouldFail": true
+    "section": "Block quotes"
   },
   {
     "markdown": "> ```\nfoo\n```\n",
     "html": "<blockquote>\n<pre><code></code></pre>\n</blockquote>\n<p>foo</p>\n<pre><code></code></pre>\n",
     "example": 237,
     "start_line": 3851,
     "end_line": 3861,
-    "section": "Block quotes",
-    "shouldFail": true
+    "section": "Block quotes"
   },
   {
     "markdown": "> foo\n    - bar\n",

diff --git a/test/unit/marked.test.js b/test/unit/marked.test.js
@@ -18,7 +18,7 @@ describe('marked unit', () => {
 
       assert.strictEqual(tokens[0].type, 'paragraph');
       assert.strictEqual(tokens[2].tokens[0].type, 'paragraph');
-      assert.strictEqual(tokens[3].items[0].tokens[0].type, 'text');
+      assert.strictEqual(tokens[4].items[0].tokens[0].type, 'text');
     });
   });
 
@@ -924,6 +924,7 @@ br
         ['blockquote', '> blockquote'],
         ['paragraph', 'blockquote'],
         ['text', 'blockquote'],
+        ['space', ''],
         ['list', '- list'],
         ['list_item', '- list'],
         ['text', 'list'],