Issue 848 (#864)

* #850

changelog.md

@@ -27,6 +27,13 @@
 - [Issue 843](https://github.com/jackdewinter/pymarkdown/issues/843)
   - whitespace check not being suspended for one check caused the html block
     not to be closed
+- [Issue 848](https://github.com/jackdewinter/pymarkdown/issues/848)
+  - indent spacing within containers causing tab to not be changed back
+    properly
+- [Issue 850](https://github.com/jackdewinter/pymarkdown/issues/850)
+  - some of the lines we adding whitespace for both the bq indent (already taken
+    care of) and the list, resulting in too many spaces
+  - tab support added to those cases
 - [Issue 852](https://github.com/jackdewinter/pymarkdown/issues/852)
   - fixed bad tokenization. previous fix was improper, causing strings to
     be improperly indexed into to fix spacing issue

publish/coverage.json

@@ -2,12 +2,12 @@
     "projectName": "pymarkdown",
     "reportSource": "pytest",
     "branchLevel": {
-        "totalMeasured": 4199,
-        "totalCovered": 4199
+        "totalMeasured": 4211,
+        "totalCovered": 4211
     },
     "lineLevel": {
-        "totalMeasured": 17663,
-        "totalCovered": 17663
+        "totalMeasured": 17701,
+        "totalCovered": 17701
     }
 }
 

publish/pylint_suppression.json

@@ -168,7 +168,7 @@
             "too-many-arguments": 4
         },
         "pymarkdown/leaf_blocks/leaf_block_helper.py": {
-            "too-many-arguments": 1
+            "too-many-arguments": 3
         },
         "pymarkdown/leaf_blocks/leaf_block_processor.py": {
             "too-many-arguments": 1
@@ -476,7 +476,7 @@
         "too-many-instance-attributes": 23,
         "too-many-public-methods": 4,
         "too-few-public-methods": 40,
-        "too-many-arguments": 202,
+        "too-many-arguments": 204,
         "too-many-locals": 25,
         "chained-comparison": 1,
         "too-many-boolean-expressions": 2,

publish/test-results.json

@@ -708,10 +708,10 @@
         },
         {
             "name": "test.gfm.test_markdown_whitespace_html",
-            "totalTests": 71,
+            "totalTests": 73,
             "failedTests": 0,
             "errorTests": 0,
-            "skippedTests": 13,
+            "skippedTests": 10,
             "elapsedTimeInMilliseconds": 0
         },
         {

pymarkdown/leaf_blocks/leaf_block_helper.py

@@ -7,6 +7,8 @@
 from pymarkdown.general.parser_helper import ParserHelper
 from pymarkdown.general.parser_logger import ParserLogger
 from pymarkdown.general.parser_state import ParserState
+from pymarkdown.general.tab_helper import TabHelper
+from pymarkdown.tokens.block_quote_markdown_token import BlockQuoteMarkdownToken
 from pymarkdown.tokens.list_start_markdown_token import ListStartMarkdownToken
 from pymarkdown.tokens.markdown_token import MarkdownToken
 from pymarkdown.tokens.stack_token import ListStackToken, StackToken
@@ -29,6 +31,8 @@ def correct_for_leaf_block_start_in_list(
         was_token_already_added_to_stack: bool = True,
         delay_tab_match: bool = False,
         alt_removed_chars_at_start: Optional[int] = None,
+        is_html: bool = False,
+        original_line: Optional[str] = None,
     ) -> None:
         """
         Check to see that if a paragraph has been closed within a list and
@@ -76,6 +80,8 @@ def correct_for_leaf_block_start_in_list(
             html_tokens,
             delay_tab_match,
             alt_removed_chars_at_start,
+            is_html,
+            original_line,
         )
 
         if was_token_already_added_to_stack:
@@ -92,13 +98,103 @@ def correct_for_leaf_block_start_in_list(
         # assert False
 
     # pylint: enable=too-many-arguments
+
+    @staticmethod
+    def __handle_leaf_start_adjust_tab(
+        parser_state: ParserState,
+        original_line: str,
+        leading_chars_at_start: str,
+        indent_count: int,
+        used_indent: str,
+    ) -> str:
+        after_index_index = len(leading_chars_at_start) + indent_count
+        used_indent_text = leading_chars_at_start + used_indent
+        assert parser_state.original_line_to_parse is not None
+        reconstructed_line = (
+            used_indent_text + parser_state.original_line_to_parse[after_index_index:]
+        )
+        detabified_original_line = TabHelper.detabify_string(original_line)
+        assert detabified_original_line == reconstructed_line
+        keep_going = True
+        detab_index = 1
+        while keep_going:
+            orig_prefix = original_line[:detab_index]
+            detab_orig_prefix = TabHelper.detabify_string(orig_prefix)
+            keep_going = len(detab_orig_prefix) < len(used_indent_text)
+            if keep_going:
+                detab_index += 1
+        assert len(detab_orig_prefix) == len(used_indent_text)
+        return orig_prefix[len(leading_chars_at_start) :]
+
+    # pylint: disable=too-many-arguments
+    @staticmethod
+    def __handle_leaf_start_adjust(
+        parser_state: ParserState,
+        delay_tab_match: bool,
+        alt_removed_chars_at_start: Optional[int],
+        is_html: bool,
+        original_line: Optional[str],
+        list_stack_token: ListStackToken,
+        removed_chars_at_start: int,
+    ) -> None:
+        if delay_tab_match:
+            used_indent = ""
+        else:
+            leading_chars_at_start = ""
+            if alt_removed_chars_at_start is not None:
+                indent_count = alt_removed_chars_at_start
+            else:
+                if is_html:
+                    last_bq_index = parser_state.find_last_block_quote_on_stack()
+                    if last_bq_index > 0:
+                        last_bq_token = cast(
+                            BlockQuoteMarkdownToken,
+                            parser_state.token_stack[
+                                last_bq_index
+                            ].matching_markdown_token,
+                        )
+                        assert last_bq_token.bleading_spaces is not None
+                        leading_chars_at_start = last_bq_token.bleading_spaces
+                        last_newline_index = leading_chars_at_start.rfind("\n")
+                        assert last_newline_index != -1
+                        leading_chars_at_start = leading_chars_at_start[
+                            last_newline_index + 1 :
+                        ]
+                        removed_chars_at_start -= len(leading_chars_at_start)
+
+                indent_count = removed_chars_at_start
+            used_indent = ParserHelper.repeat_string(" ", indent_count)
+            if (
+                is_html
+                and original_line is not None
+                and leading_chars_at_start
+                and "\t" in original_line
+            ):
+                used_indent = LeafBlockHelper.__handle_leaf_start_adjust_tab(
+                    parser_state,
+                    original_line,
+                    leading_chars_at_start,
+                    indent_count,
+                    used_indent,
+                )
+        assert list_stack_token.matching_markdown_token is not None
+        list_markdown_token = cast(
+            ListStartMarkdownToken, list_stack_token.matching_markdown_token
+        )
+        list_markdown_token.add_leading_spaces(used_indent)
+
+    # pylint: enable=too-many-arguments
+
+    # pylint: disable=too-many-arguments
     @staticmethod
     def __handle_leaf_start(
         parser_state: ParserState,
         removed_chars_at_start: int,
         html_tokens: List[MarkdownToken],
         delay_tab_match: bool,
         alt_removed_chars_at_start: Optional[int],
+        is_html: bool,
+        original_line: Optional[str] = None,
     ) -> None:
         POGGER.debug(
             ">>correct_for_leaf_block_start_in_list>>stack>>$>>",
@@ -146,20 +242,17 @@ def __handle_leaf_start(
             POGGER.debug(">>delay_tab_match>>$>>", delay_tab_match)
             # assert not delay_tab_match
             if adjust_with_leading_spaces:
-                if delay_tab_match:
-                    used_indent = ""
-                else:
-                    indent_count = (
-                        alt_removed_chars_at_start
-                        if alt_removed_chars_at_start is not None
-                        else removed_chars_at_start
-                    )
-                    used_indent = ParserHelper.repeat_string(" ", indent_count)
-                assert list_stack_token.matching_markdown_token is not None
-                list_markdown_token = cast(
-                    ListStartMarkdownToken, list_stack_token.matching_markdown_token
+                LeafBlockHelper.__handle_leaf_start_adjust(
+                    parser_state,
+                    delay_tab_match,
+                    alt_removed_chars_at_start,
+                    is_html,
+                    original_line,
+                    list_stack_token,
+                    removed_chars_at_start,
                 )
-                list_markdown_token.add_leading_spaces(used_indent)
+
+    # pylint: enable=too-many-arguments
 
     @staticmethod
     def extract_markdown_tokens_back_to_blank_line(

pymarkdown/leaf_blocks/leaf_block_processor.py

@@ -132,6 +132,8 @@ def handle_html_block(
                     old_top_of_stack,
                     html_tokens,
                     alt_removed_chars_at_start=alt_removed_chars_at_start,
+                    is_html=True,
+                    original_line=grab_bag.original_line,
                 )
             new_tokens.extend(html_tokens)
         if parser_state.token_stack[-1].is_html_block: