fix: preserve internal tabs in list item content

src/Tokenizer.ts

@@ -293,7 +293,11 @@ export class _Tokenizer<ParserOutput = string, RendererOutput = string> {
         raw = cap[0];
         src = src.substring(raw.length);
 
-        let line = expandTabs(cap[2].split('\n', 1)[0], cap[1].length);
+        const firstLine = cap[2].split('\n', 1)[0];
+        const bulletIndent = cap[1].length;
+        let line = this.options.pedantic
+          ? expandTabs(firstLine, bulletIndent)
+          : firstLine.replace(this.rules.other.leadingSpaceTab, whitespace => expandTabs(whitespace, bulletIndent));
         let nextLine = src.split('\n', 1)[0];
         let blankLine = !line.trim();
 
@@ -302,12 +306,12 @@ export class _Tokenizer<ParserOutput = string, RendererOutput = string> {
           indent = 2;
           itemContents = line.trimStart();
         } else if (blankLine) {
-          indent = cap[1].length + 1;
+          indent = bulletIndent + 1;
         } else {
           indent = line.search(this.rules.other.nonSpaceChar); // Find first non-space char
           indent = indent > 4 ? 1 : indent; // Treat indented code blocks (> 4 spaces) as having only 1 indent
           itemContents = line.slice(indent);
-          indent += cap[1].length;
+          indent += bulletIndent;
         }
 
         if (blankLine && this.rules.other.blankLine.test(nextLine)) { // Items begin with at most one blank line
@@ -335,7 +339,7 @@ export class _Tokenizer<ParserOutput = string, RendererOutput = string> {
               nextLine = nextLine.replace(this.rules.other.listReplaceNesting, '  ');
               nextLineWithoutTabs = nextLine;
             } else {
-              nextLineWithoutTabs = nextLine.replace(this.rules.other.tabCharGlobal, '    ');
+              nextLineWithoutTabs = nextLine.replace(this.rules.other.leadingSpaceTab, whitespace => whitespace.replace(this.rules.other.tabCharGlobal, '    '));
             }
 
             // End list item if found code fences

src/rules.ts

@@ -52,6 +52,7 @@ export const other = {
   nonSpaceChar: /[^ ]/,
   newLineCharGlobal: /\n/g,
   tabCharGlobal: /\t/g,
+  leadingSpaceTab: /^[ \t]+/,
   multipleSpaceGlobal: /\s+/g,
   blankLine: /^[ \t]*$/,
   doubleBlankLine: /\n[ \t]*\n[ \t]*$/,

test/specs/new/list_content_tabs_continuation_code_span_commonmark.html

@@ -0,0 +1,5 @@
+<ul>
+<li><p>item</p>
+<p><code>foo	bar</code></p>
+</li>
+</ul>

test/specs/new/list_content_tabs_continuation_code_span_commonmark.md

@@ -0,0 +1,7 @@
+---
+gfm: false
+renderExact: true
+---
+- item
+
+  `foo	bar`

test/specs/new/list_content_tabs_continuation_code_span_gfm.html

@@ -0,0 +1,5 @@
+<ul>
+<li><p>item</p>
+<p><code>foo	bar</code></p>
+</li>
+</ul>

test/specs/new/list_content_tabs_continuation_code_span_gfm.md

@@ -0,0 +1,7 @@
+---
+gfm: true
+renderExact: true
+---
+- item
+
+  `foo	bar`

test/specs/new/list_content_tabs_fence_commonmark.html

@@ -0,0 +1,5 @@
+<ul>
+<li><pre><code>foo	bar
+</code></pre>
+</li>
+</ul>

test/specs/new/list_content_tabs_fence_commonmark.md

@@ -0,0 +1,7 @@
+---
+gfm: false
+renderExact: true
+---
+- ```
+  foo	bar
+  ```

test/specs/new/list_content_tabs_fence_gfm.html

@@ -0,0 +1,5 @@
+<ul>
+<li><pre><code>foo	bar
+</code></pre>
+</li>
+</ul>

test/specs/new/list_content_tabs_fence_gfm.md

@@ -0,0 +1,7 @@
+---
+gfm: true
+renderExact: true
+---
+- ```
+  foo	bar
+  ```

test/specs/new/list_content_tabs_first_line_code_span_commonmark.html

@@ -0,0 +1,3 @@
+<ul>
+<li><code>foo	bar</code></li>
+</ul>

test/specs/new/list_content_tabs_first_line_code_span_commonmark.md

@@ -0,0 +1,5 @@
+---
+gfm: false
+renderExact: true
+---
+- `foo	bar`

test/specs/new/list_content_tabs_first_line_code_span_gfm.html

@@ -0,0 +1,3 @@
+<ul>
+<li><code>foo	bar</code></li>
+</ul>

test/specs/new/list_content_tabs_first_line_code_span_gfm.md

@@ -0,0 +1,5 @@
+---
+gfm: true
+renderExact: true
+---
+- `foo	bar`

test/specs/new/list_content_tabs_nested_fence_commonmark.html

@@ -0,0 +1,8 @@
+<ul>
+<li>outer<ul>
+<li><pre><code>foo	bar
+</code></pre>
+</li>
+</ul>
+</li>
+</ul>

test/specs/new/list_content_tabs_nested_fence_commonmark.md

@@ -0,0 +1,8 @@
+---
+gfm: false
+renderExact: true
+---
+- outer
+  - ```
+    foo	bar
+    ```

test/specs/new/list_content_tabs_nested_fence_gfm.html

@@ -0,0 +1,8 @@
+<ul>
+<li>outer<ul>
+<li><pre><code>foo	bar
+</code></pre>
+</li>
+</ul>
+</li>
+</ul>

test/specs/new/list_content_tabs_nested_fence_gfm.md

@@ -0,0 +1,8 @@
+---
+gfm: true
+renderExact: true
+---
+- outer
+  - ```
+    foo	bar
+    ```

test/specs/new/list_content_tabs_ordered_fence_commonmark.html

@@ -0,0 +1,5 @@
+<ol>
+<li><pre><code>foo	bar
+</code></pre>
+</li>
+</ol>

test/specs/new/list_content_tabs_ordered_fence_commonmark.md

@@ -0,0 +1,7 @@
+---
+gfm: false
+renderExact: true
+---
+1. ```
+   foo	bar
+   ```

test/specs/new/list_content_tabs_ordered_fence_gfm.html

@@ -0,0 +1,5 @@
+<ol>
+<li><pre><code>foo	bar
+</code></pre>
+</li>
+</ol>

test/specs/new/list_content_tabs_ordered_fence_gfm.md

@@ -0,0 +1,7 @@
+---
+gfm: true
+renderExact: true
+---
+1. ```
+   foo	bar
+   ```

test/specs/new/list_content_tabs_repeated_tabs_commonmark.html

@@ -0,0 +1,5 @@
+<ul>
+<li><pre><code>foo		bar
+</code></pre>
+</li>
+</ul>

test/specs/new/list_content_tabs_repeated_tabs_commonmark.md

@@ -0,0 +1,7 @@
+---
+gfm: false
+renderExact: true
+---
+- ~~~
+  foo		bar
+  ~~~

test/specs/new/list_content_tabs_repeated_tabs_gfm.html

@@ -0,0 +1,5 @@
+<ul>
+<li><pre><code>foo		bar
+</code></pre>
+</li>
+</ul>

test/specs/new/list_content_tabs_repeated_tabs_gfm.md

@@ -0,0 +1,7 @@
+---
+gfm: true
+renderExact: true
+---
+- ~~~
+  foo		bar
+  ~~~

test/specs/new/list_content_tabs_structural_indentation_tab_commonmark.html

@@ -0,0 +1,5 @@
+<ul>
+<li><pre><code>  foo	bar
+</code></pre>
+</li>
+</ul>

test/specs/new/list_content_tabs_structural_indentation_tab_commonmark.md

@@ -0,0 +1,7 @@
+---
+gfm: false
+renderExact: true
+---
+- ```
+	foo	bar
+  ```

test/specs/new/list_content_tabs_structural_indentation_tab_gfm.html

@@ -0,0 +1,5 @@
+<ul>
+<li><pre><code>  foo	bar
+</code></pre>
+</li>
+</ul>

test/specs/new/list_content_tabs_structural_indentation_tab_gfm.md

@@ -0,0 +1,7 @@
+---
+gfm: true
+renderExact: true
+---
+- ```
+	foo	bar
+  ```

test/specs/new/list_content_tabs_tab_separated_marker_commonmark.html

@@ -0,0 +1,3 @@
+<ul>
+<li><code>foo	bar</code></li>
+</ul>

test/specs/new/list_content_tabs_tab_separated_marker_commonmark.md

@@ -0,0 +1,5 @@
+---
+gfm: false
+renderExact: true
+---
+-	`foo	bar`

test/specs/new/list_content_tabs_tab_separated_marker_gfm.html

@@ -0,0 +1,3 @@
+<ul>
+<li><code>foo	bar</code></li>
+</ul>

test/specs/new/list_content_tabs_tab_separated_marker_gfm.md

@@ -0,0 +1,5 @@
+---
+gfm: true
+renderExact: true
+---
+-	`foo	bar`