Refactor micromark parse code to stop using micromark's TokenizeConte…

…xt.sliceSerialize (less well supported) in favor of Token.start/end.offset.
DavidAnson · Feb 8, 2025 · 3cbe1cb · 3cbe1cb
1 parent b23fc96
commit 3cbe1cb
Show file tree

Hide file tree

Showing 6 changed files with 71 additions and 25 deletions.
diff --git a/helpers/helpers.cjs b/helpers/helpers.cjs
@@ -378,6 +378,7 @@ module.exports.frontMatterHasTitle =
  */
 function getReferenceLinkImageData(tokens) {
   const normalizeReference = (s) => s.toLowerCase().trim().replace(/\s+/g, " ");
+  const getText = (t) => t?.children.filter((c) => c.type !== "blockQuotePrefix").map((c) => c.text).join("");
   const references = new Map();
   const shortcuts = new Map();
   const addReferenceToDictionary = (token, label, isShortcut) => {
@@ -449,7 +450,7 @@ function getReferenceLinkImageData(tokens) {
           const isFullOrCollapsed = (token.children.length === 2) && !token.children.some((t) => t.type === "resource");
           const [ labelText ] = micromark.getDescendantsByType(token, [ "label", "labelText" ]);
           const [ referenceString ] = micromark.getDescendantsByType(token, [ "reference", "referenceString" ]);
-          let label = labelText?.text;
+          let label = getText(labelText);
           // Identify if footnote
           if (!isShortcut && !isFullOrCollapsed) {
             const [ footnoteCallMarker, footnoteCallString ] = token.children.filter(
@@ -462,7 +463,7 @@ function getReferenceLinkImageData(tokens) {
           }
           // Track link (handle shortcuts separately due to ambiguity in "text [text] text")
           if (isShortcut || isFullOrCollapsed) {
-            addReferenceToDictionary(token, referenceString?.text || label, isShortcut);
+            addReferenceToDictionary(token, getText(referenceString) || label, isShortcut);
           }
         }
         break;

diff --git a/lib/micromark-parse.mjs b/lib/micromark-parse.mjs
@@ -20,6 +20,17 @@ import { flatTokensSymbol, htmlFlowSymbol, newLineRe } from "../helpers/shared.c
 /** @typedef {import("markdownlint").MicromarkToken} MicromarkToken */
 /** @typedef {import("./micromark-types.d.mts")} */
 
+/**
+ * Gets the Markdown text for a Micromark token.
+ *
+ * @param {string} markdown Markdown content.
+ * @param {Token} token Micromark token.
+ * @returns {string} Token text.
+ */
+function getText(markdown, token) {
+  return markdown.slice(token.start.offset, token.end.offset);
+}
+
 /**
  * Parse options.
  *
@@ -126,7 +137,7 @@ export function getEvents(
         // Create artificial event list and replicate content
         const text = eventsToReplicate
           .filter((event) => event[0] === "enter")
-          .map((event) => tokenizeContext.sliceSerialize(event[1]))
+          .map((event) => getText(markdown, event[1]))
           .join("")
           .trim();
         if ((text.length > 0) && !text.includes("]")) {
@@ -221,11 +232,11 @@ function parseInternal(
   let lines = null;
   let skipHtmlFlowChildren = false;
   for (const event of events) {
-    const [ kind, token, context ] = event;
+    const [ kind, token ] = event;
     const { type, start, end } = token;
     const { "column": startColumn, "line": startLine } = start;
     const { "column": endColumn, "line": endLine } = end;
-    const text = context.sliceSerialize(token);
+    const text = getText(markdown, token);
     if ((kind === "enter") && !skipHtmlFlowChildren) {
       const previous = current;
       history.push(previous);

diff --git a/test/snapshots/markdownlint-test-custom-rules.mjs.md b/test/snapshots/markdownlint-test-custom-rules.mjs.md
@@ -4860,7 +4860,7 @@ Generated by [AVA](https://avajs.dev).
             parent: [Circular],
             startColumn: 3,
             startLine: 21,
-            text: '',
+            text: '> Nested',
             type: 'blockQuote',
           },
         ],
@@ -4869,7 +4869,8 @@ Generated by [AVA](https://avajs.dev).
         parent: null,
         startColumn: 1,
         startLine: 20,
-        text: '',
+        text: `> Block quote␊
+        > > Nested`,
         type: 'blockQuote',
       },
       {
@@ -5149,7 +5150,7 @@ Generated by [AVA](https://avajs.dev).
                     startColumn: 5,
                     startLine: 25,
                     text: `Items␊
-                    Indented`,
+                        Indented`,
                     type: 'paragraph',
                   },
                 ],
@@ -5159,7 +5160,7 @@ Generated by [AVA](https://avajs.dev).
                 startColumn: 5,
                 startLine: 25,
                 text: `Items␊
-                Indented`,
+                    Indented`,
                 type: 'content',
               },
               {
@@ -5242,7 +5243,10 @@ Generated by [AVA](https://avajs.dev).
             parent: [Circular],
             startColumn: 3,
             startLine: 25,
-            text: '',
+            text: `- Items␊
+                Indented␊
+            ␊
+                Content`,
             type: 'listUnordered',
           },
         ],
@@ -5251,7 +5255,12 @@ Generated by [AVA](https://avajs.dev).
         parent: null,
         startColumn: 1,
         startLine: 23,
-        text: '',
+        text: `- Unordered␊
+        - List␊
+          - Items␊
+            Indented␊
+        ␊
+            Content`,
         type: 'listUnordered',
       },
       {
@@ -5561,7 +5570,7 @@ Generated by [AVA](https://avajs.dev).
                     startColumn: 7,
                     startLine: 32,
                     text: `Items␊
-                    Indented`,
+                          Indented`,
                     type: 'paragraph',
                   },
                 ],
@@ -5571,7 +5580,7 @@ Generated by [AVA](https://avajs.dev).
                 startColumn: 7,
                 startLine: 32,
                 text: `Items␊
-                Indented`,
+                      Indented`,
                 type: 'content',
               },
               {
@@ -5654,7 +5663,10 @@ Generated by [AVA](https://avajs.dev).
             parent: [Circular],
             startColumn: 4,
             startLine: 32,
-            text: '',
+            text: `1. Items␊
+                  Indented␊
+            ␊
+                  Content`,
             type: 'listOrdered',
           },
         ],
@@ -5663,7 +5675,12 @@ Generated by [AVA](https://avajs.dev).
         parent: null,
         startColumn: 1,
         startLine: 30,
-        text: '',
+        text: `1. Ordered␊
+        2. List␊
+           1. Items␊
+              Indented␊
+        ␊
+              Content`,
         type: 'listOrdered',
       },
       {

diff --git a/test/snapshots/markdownlint-test-custom-rules.mjs.snap b/test/snapshots/markdownlint-test-custom-rules.mjs.snap
diff --git a/test/snapshots/markdownlint-test-micromark.mjs.md b/test/snapshots/markdownlint-test-micromark.mjs.md
@@ -1798,7 +1798,7 @@ Generated by [AVA](https://avajs.dev).
             parent: [Circular],
             startColumn: 3,
             startLine: 21,
-            text: '',
+            text: '> Nested',
             type: 'blockQuote',
           },
         ],
@@ -1807,7 +1807,8 @@ Generated by [AVA](https://avajs.dev).
         parent: null,
         startColumn: 1,
         startLine: 20,
-        text: '',
+        text: `> Block quote␊
+        > > Nested`,
         type: 'blockQuote',
       },
       {
@@ -2087,7 +2088,7 @@ Generated by [AVA](https://avajs.dev).
                     startColumn: 5,
                     startLine: 25,
                     text: `Items␊
-                    Indented`,
+                        Indented`,
                     type: 'paragraph',
                   },
                 ],
@@ -2097,7 +2098,7 @@ Generated by [AVA](https://avajs.dev).
                 startColumn: 5,
                 startLine: 25,
                 text: `Items␊
-                Indented`,
+                    Indented`,
                 type: 'content',
               },
               {
@@ -2180,7 +2181,10 @@ Generated by [AVA](https://avajs.dev).
             parent: [Circular],
             startColumn: 3,
             startLine: 25,
-            text: '',
+            text: `- Items␊
+                Indented␊
+            ␊
+                Content`,
             type: 'listUnordered',
           },
         ],
@@ -2189,7 +2193,12 @@ Generated by [AVA](https://avajs.dev).
         parent: null,
         startColumn: 1,
         startLine: 23,
-        text: '',
+        text: `- Unordered␊
+        - List␊
+          - Items␊
+            Indented␊
+        ␊
+            Content`,
         type: 'listUnordered',
       },
       {
@@ -2499,7 +2508,7 @@ Generated by [AVA](https://avajs.dev).
                     startColumn: 7,
                     startLine: 32,
                     text: `Items␊
-                    Indented`,
+                          Indented`,
                     type: 'paragraph',
                   },
                 ],
@@ -2509,7 +2518,7 @@ Generated by [AVA](https://avajs.dev).
                 startColumn: 7,
                 startLine: 32,
                 text: `Items␊
-                Indented`,
+                      Indented`,
                 type: 'content',
               },
               {
@@ -2592,7 +2601,10 @@ Generated by [AVA](https://avajs.dev).
             parent: [Circular],
             startColumn: 4,
             startLine: 32,
-            text: '',
+            text: `1. Items␊
+                  Indented␊
+            ␊
+                  Content`,
             type: 'listOrdered',
           },
         ],
@@ -2601,7 +2613,12 @@ Generated by [AVA](https://avajs.dev).
         parent: null,
         startColumn: 1,
         startLine: 30,
-        text: '',
+        text: `1. Ordered␊
+        2. List␊
+           1. Items␊
+              Indented␊
+        ␊
+              Content`,
         type: 'listOrdered',
       },
       {

diff --git a/test/snapshots/markdownlint-test-micromark.mjs.snap b/test/snapshots/markdownlint-test-micromark.mjs.snap