diff --git a/src/Tokenizer.ts b/src/Tokenizer.ts index 369bb15fe1..226b0f4821 100644 --- a/src/Tokenizer.ts +++ b/src/Tokenizer.ts @@ -293,7 +293,11 @@ export class _Tokenizer { raw = cap[0]; src = src.substring(raw.length); - let line = expandTabs(cap[2].split('\n', 1)[0], cap[1].length); + const firstLine = cap[2].split('\n', 1)[0]; + const bulletIndent = cap[1].length; + let line = this.options.pedantic + ? expandTabs(firstLine, bulletIndent) + : firstLine.replace(this.rules.other.leadingSpaceTab, whitespace => expandTabs(whitespace, bulletIndent)); let nextLine = src.split('\n', 1)[0]; let blankLine = !line.trim(); @@ -302,12 +306,12 @@ export class _Tokenizer { indent = 2; itemContents = line.trimStart(); } else if (blankLine) { - indent = cap[1].length + 1; + indent = bulletIndent + 1; } else { indent = line.search(this.rules.other.nonSpaceChar); // Find first non-space char indent = indent > 4 ? 1 : indent; // Treat indented code blocks (> 4 spaces) as having only 1 indent itemContents = line.slice(indent); - indent += cap[1].length; + indent += bulletIndent; } if (blankLine && this.rules.other.blankLine.test(nextLine)) { // Items begin with at most one blank line @@ -335,7 +339,7 @@ export class _Tokenizer { nextLine = nextLine.replace(this.rules.other.listReplaceNesting, ' '); nextLineWithoutTabs = nextLine; } else { - nextLineWithoutTabs = nextLine.replace(this.rules.other.tabCharGlobal, ' '); + nextLineWithoutTabs = nextLine.replace(this.rules.other.leadingSpaceTab, whitespace => whitespace.replace(this.rules.other.tabCharGlobal, ' ')); } // End list item if found code fences diff --git a/src/rules.ts b/src/rules.ts index 33643c5f90..0bc416c177 100644 --- a/src/rules.ts +++ b/src/rules.ts @@ -52,6 +52,7 @@ export const other = { nonSpaceChar: /[^ ]/, newLineCharGlobal: /\n/g, tabCharGlobal: /\t/g, + leadingSpaceTab: /^[ \t]+/, multipleSpaceGlobal: /\s+/g, blankLine: /^[ \t]*$/, doubleBlankLine: /\n[ \t]*\n[ \t]*$/, diff --git a/test/specs/new/list_content_tabs_continuation_code_span_commonmark.html b/test/specs/new/list_content_tabs_continuation_code_span_commonmark.html new file mode 100644 index 0000000000..b41a4f57cf --- /dev/null +++ b/test/specs/new/list_content_tabs_continuation_code_span_commonmark.html @@ -0,0 +1,5 @@ +
    +

  • item

    +

    foo bar

    +
    • +
diff --git a/test/specs/new/list_content_tabs_continuation_code_span_commonmark.md b/test/specs/new/list_content_tabs_continuation_code_span_commonmark.md new file mode 100644 index 0000000000..77b7a3b170 --- /dev/null +++ b/test/specs/new/list_content_tabs_continuation_code_span_commonmark.md @@ -0,0 +1,7 @@ +--- +gfm: false +renderExact: true +--- +- item + + `foo bar` diff --git a/test/specs/new/list_content_tabs_continuation_code_span_gfm.html b/test/specs/new/list_content_tabs_continuation_code_span_gfm.html new file mode 100644 index 0000000000..b41a4f57cf --- /dev/null +++ b/test/specs/new/list_content_tabs_continuation_code_span_gfm.html @@ -0,0 +1,5 @@ +
    +

  • item

    +

    foo bar

    +
    • +
diff --git a/test/specs/new/list_content_tabs_continuation_code_span_gfm.md b/test/specs/new/list_content_tabs_continuation_code_span_gfm.md new file mode 100644 index 0000000000..5d5e6d7a05 --- /dev/null +++ b/test/specs/new/list_content_tabs_continuation_code_span_gfm.md @@ -0,0 +1,7 @@ +--- +gfm: true +renderExact: true +--- +- item + + `foo bar` diff --git a/test/specs/new/list_content_tabs_fence_commonmark.html b/test/specs/new/list_content_tabs_fence_commonmark.html new file mode 100644 index 0000000000..50976d2f7f --- /dev/null +++ b/test/specs/new/list_content_tabs_fence_commonmark.html @@ -0,0 +1,5 @@ +
    +
  • foo	bar
+
    +
    • +
diff --git a/test/specs/new/list_content_tabs_fence_commonmark.md b/test/specs/new/list_content_tabs_fence_commonmark.md new file mode 100644 index 0000000000..35f83bc057 --- /dev/null +++ b/test/specs/new/list_content_tabs_fence_commonmark.md @@ -0,0 +1,7 @@ +--- +gfm: false +renderExact: true +--- +- ``` + foo bar + ``` diff --git a/test/specs/new/list_content_tabs_fence_gfm.html b/test/specs/new/list_content_tabs_fence_gfm.html new file mode 100644 index 0000000000..50976d2f7f --- /dev/null +++ b/test/specs/new/list_content_tabs_fence_gfm.html @@ -0,0 +1,5 @@ +
    +
  • foo	bar
+
    +
    • +
diff --git a/test/specs/new/list_content_tabs_fence_gfm.md b/test/specs/new/list_content_tabs_fence_gfm.md new file mode 100644 index 0000000000..bd9550fcd5 --- /dev/null +++ b/test/specs/new/list_content_tabs_fence_gfm.md @@ -0,0 +1,7 @@ +--- +gfm: true +renderExact: true +--- +- ``` + foo bar + ``` diff --git a/test/specs/new/list_content_tabs_first_line_code_span_commonmark.html b/test/specs/new/list_content_tabs_first_line_code_span_commonmark.html new file mode 100644 index 0000000000..e3f83e9411 --- /dev/null +++ b/test/specs/new/list_content_tabs_first_line_code_span_commonmark.html @@ -0,0 +1,3 @@ +
    +
  • foo bar
    • +
diff --git a/test/specs/new/list_content_tabs_first_line_code_span_commonmark.md b/test/specs/new/list_content_tabs_first_line_code_span_commonmark.md new file mode 100644 index 0000000000..4835e46b52 --- /dev/null +++ b/test/specs/new/list_content_tabs_first_line_code_span_commonmark.md @@ -0,0 +1,5 @@ +--- +gfm: false +renderExact: true +--- +- `foo bar` diff --git a/test/specs/new/list_content_tabs_first_line_code_span_gfm.html b/test/specs/new/list_content_tabs_first_line_code_span_gfm.html new file mode 100644 index 0000000000..e3f83e9411 --- /dev/null +++ b/test/specs/new/list_content_tabs_first_line_code_span_gfm.html @@ -0,0 +1,3 @@ +
    +
  • foo bar
    • +
diff --git a/test/specs/new/list_content_tabs_first_line_code_span_gfm.md b/test/specs/new/list_content_tabs_first_line_code_span_gfm.md new file mode 100644 index 0000000000..f0644f7c2b --- /dev/null +++ b/test/specs/new/list_content_tabs_first_line_code_span_gfm.md @@ -0,0 +1,5 @@ +--- +gfm: true +renderExact: true +--- +- `foo bar` diff --git a/test/specs/new/list_content_tabs_nested_fence_commonmark.html b/test/specs/new/list_content_tabs_nested_fence_commonmark.html new file mode 100644 index 0000000000..4430fb4681 --- /dev/null +++ b/test/specs/new/list_content_tabs_nested_fence_commonmark.html @@ -0,0 +1,8 @@ +
    +
  • outer
      +
    • foo	bar
+
      +
      • +
    +
    • +
diff --git a/test/specs/new/list_content_tabs_nested_fence_commonmark.md b/test/specs/new/list_content_tabs_nested_fence_commonmark.md new file mode 100644 index 0000000000..5ed74938a8 --- /dev/null +++ b/test/specs/new/list_content_tabs_nested_fence_commonmark.md @@ -0,0 +1,8 @@ +--- +gfm: false +renderExact: true +--- +- outer + - ``` + foo bar + ``` diff --git a/test/specs/new/list_content_tabs_nested_fence_gfm.html b/test/specs/new/list_content_tabs_nested_fence_gfm.html new file mode 100644 index 0000000000..4430fb4681 --- /dev/null +++ b/test/specs/new/list_content_tabs_nested_fence_gfm.html @@ -0,0 +1,8 @@ +
    +
  • outer
      +
    • foo	bar
+
      +
      • +
    +
    • +
diff --git a/test/specs/new/list_content_tabs_nested_fence_gfm.md b/test/specs/new/list_content_tabs_nested_fence_gfm.md new file mode 100644 index 0000000000..3e1e62079b --- /dev/null +++ b/test/specs/new/list_content_tabs_nested_fence_gfm.md @@ -0,0 +1,8 @@ +--- +gfm: true +renderExact: true +--- +- outer + - ``` + foo bar + ``` diff --git a/test/specs/new/list_content_tabs_ordered_fence_commonmark.html b/test/specs/new/list_content_tabs_ordered_fence_commonmark.html new file mode 100644 index 0000000000..345694acb1 --- /dev/null +++ b/test/specs/new/list_content_tabs_ordered_fence_commonmark.html @@ -0,0 +1,5 @@ +
    +
  1. foo	bar
+
    +
    2. +
diff --git a/test/specs/new/list_content_tabs_ordered_fence_commonmark.md b/test/specs/new/list_content_tabs_ordered_fence_commonmark.md new file mode 100644 index 0000000000..d2e7526d48 --- /dev/null +++ b/test/specs/new/list_content_tabs_ordered_fence_commonmark.md @@ -0,0 +1,7 @@ +--- +gfm: false +renderExact: true +--- +1. ``` + foo bar + ``` diff --git a/test/specs/new/list_content_tabs_ordered_fence_gfm.html b/test/specs/new/list_content_tabs_ordered_fence_gfm.html new file mode 100644 index 0000000000..345694acb1 --- /dev/null +++ b/test/specs/new/list_content_tabs_ordered_fence_gfm.html @@ -0,0 +1,5 @@ +
    +
  1. foo	bar
+
    +
    2. +
diff --git a/test/specs/new/list_content_tabs_ordered_fence_gfm.md b/test/specs/new/list_content_tabs_ordered_fence_gfm.md new file mode 100644 index 0000000000..7795e36c29 --- /dev/null +++ b/test/specs/new/list_content_tabs_ordered_fence_gfm.md @@ -0,0 +1,7 @@ +--- +gfm: true +renderExact: true +--- +1. ``` + foo bar + ``` diff --git a/test/specs/new/list_content_tabs_repeated_tabs_commonmark.html b/test/specs/new/list_content_tabs_repeated_tabs_commonmark.html new file mode 100644 index 0000000000..ce0ded9462 --- /dev/null +++ b/test/specs/new/list_content_tabs_repeated_tabs_commonmark.html @@ -0,0 +1,5 @@ +
    +
  • foo		bar
+
    +
    • +
diff --git a/test/specs/new/list_content_tabs_repeated_tabs_commonmark.md b/test/specs/new/list_content_tabs_repeated_tabs_commonmark.md new file mode 100644 index 0000000000..4c8ed6644e --- /dev/null +++ b/test/specs/new/list_content_tabs_repeated_tabs_commonmark.md @@ -0,0 +1,7 @@ +--- +gfm: false +renderExact: true +--- +- ~~~ + foo bar + ~~~ diff --git a/test/specs/new/list_content_tabs_repeated_tabs_gfm.html b/test/specs/new/list_content_tabs_repeated_tabs_gfm.html new file mode 100644 index 0000000000..ce0ded9462 --- /dev/null +++ b/test/specs/new/list_content_tabs_repeated_tabs_gfm.html @@ -0,0 +1,5 @@ +
    +
  • foo		bar
+
    +
    • +
diff --git a/test/specs/new/list_content_tabs_repeated_tabs_gfm.md b/test/specs/new/list_content_tabs_repeated_tabs_gfm.md new file mode 100644 index 0000000000..b542a8947d --- /dev/null +++ b/test/specs/new/list_content_tabs_repeated_tabs_gfm.md @@ -0,0 +1,7 @@ +--- +gfm: true +renderExact: true +--- +- ~~~ + foo bar + ~~~ diff --git a/test/specs/new/list_content_tabs_structural_indentation_tab_commonmark.html b/test/specs/new/list_content_tabs_structural_indentation_tab_commonmark.html new file mode 100644 index 0000000000..b66cbf4517 --- /dev/null +++ b/test/specs/new/list_content_tabs_structural_indentation_tab_commonmark.html @@ -0,0 +1,5 @@ +
    +
  •   foo	bar
+
    +
    • +
diff --git a/test/specs/new/list_content_tabs_structural_indentation_tab_commonmark.md b/test/specs/new/list_content_tabs_structural_indentation_tab_commonmark.md new file mode 100644 index 0000000000..3484b997bd --- /dev/null +++ b/test/specs/new/list_content_tabs_structural_indentation_tab_commonmark.md @@ -0,0 +1,7 @@ +--- +gfm: false +renderExact: true +--- +- ``` + foo bar + ``` diff --git a/test/specs/new/list_content_tabs_structural_indentation_tab_gfm.html b/test/specs/new/list_content_tabs_structural_indentation_tab_gfm.html new file mode 100644 index 0000000000..b66cbf4517 --- /dev/null +++ b/test/specs/new/list_content_tabs_structural_indentation_tab_gfm.html @@ -0,0 +1,5 @@ +
    +
  •   foo	bar
+
    +
    • +
diff --git a/test/specs/new/list_content_tabs_structural_indentation_tab_gfm.md b/test/specs/new/list_content_tabs_structural_indentation_tab_gfm.md new file mode 100644 index 0000000000..6c57d41a7d --- /dev/null +++ b/test/specs/new/list_content_tabs_structural_indentation_tab_gfm.md @@ -0,0 +1,7 @@ +--- +gfm: true +renderExact: true +--- +- ``` + foo bar + ``` diff --git a/test/specs/new/list_content_tabs_tab_separated_marker_commonmark.html b/test/specs/new/list_content_tabs_tab_separated_marker_commonmark.html new file mode 100644 index 0000000000..e3f83e9411 --- /dev/null +++ b/test/specs/new/list_content_tabs_tab_separated_marker_commonmark.html @@ -0,0 +1,3 @@ +
    +
  • foo bar
    • +
diff --git a/test/specs/new/list_content_tabs_tab_separated_marker_commonmark.md b/test/specs/new/list_content_tabs_tab_separated_marker_commonmark.md new file mode 100644 index 0000000000..a3965769ca --- /dev/null +++ b/test/specs/new/list_content_tabs_tab_separated_marker_commonmark.md @@ -0,0 +1,5 @@ +--- +gfm: false +renderExact: true +--- +- `foo bar` diff --git a/test/specs/new/list_content_tabs_tab_separated_marker_gfm.html b/test/specs/new/list_content_tabs_tab_separated_marker_gfm.html new file mode 100644 index 0000000000..e3f83e9411 --- /dev/null +++ b/test/specs/new/list_content_tabs_tab_separated_marker_gfm.html @@ -0,0 +1,3 @@ +
    +
  • foo bar
    • +
diff --git a/test/specs/new/list_content_tabs_tab_separated_marker_gfm.md b/test/specs/new/list_content_tabs_tab_separated_marker_gfm.md new file mode 100644 index 0000000000..4ff48075d6 --- /dev/null +++ b/test/specs/new/list_content_tabs_tab_separated_marker_gfm.md @@ -0,0 +1,5 @@ +--- +gfm: true +renderExact: true +--- +- `foo bar`