Merge pull request #274 from jackdewinter/issue-261

#264 : Dealing with …
jackdewinter · Feb 7, 2022 · ee206b4 · ee206b4
2 parents 4c7a4f5 + 001cfdf
commit ee206b4
Show file tree

Hide file tree

Showing 8 changed files with 131 additions and 33 deletions.
diff --git a/changelog.md b/changelog.md
@@ -45,6 +45,9 @@
   - parser: when checking for block quote decrease, did not have empty scenarios checked for
 - [Fixed - Issue 263](https://github.com/jackdewinter/pymarkdown/issues/263)
   - parser: with empty list items, was creating 2 blank line tokens, plus extra list indent
+- [Fixed - Issue 264](https://github.com/jackdewinter/pymarkdown/issues/264)
+  - parser: fixed issue with blending current text and original text to parse
+  - parser: cleaned up remaining issues about closing off containers too early
 - [Fixed - Issue 265](https://github.com/jackdewinter/pymarkdown/issues/265)
   - parser: fixed with work on [Issue 262](https://github.com/jackdewinter/pymarkdown/issues/262)
 - [Fixed - Issue 268](https://github.com/jackdewinter/pymarkdown/issues/268)

diff --git a/publish/coverage.json b/publish/coverage.json
@@ -2,12 +2,12 @@
     "projectName": "pymarkdown",
     "reportSource": "pytest",
     "branchLevel": {
-        "totalMeasured": 3016,
-        "totalCovered": 3016
+        "totalMeasured": 3024,
+        "totalCovered": 3024
     },
     "lineLevel": {
-        "totalMeasured": 10524,
-        "totalCovered": 10524
+        "totalMeasured": 10560,
+        "totalCovered": 10560
     }
 }
 
diff --git a/publish/pylint_suppression.json b/publish/pylint_suppression.json
@@ -18,7 +18,7 @@
             "no-self-argument": 1
         },
         "pymarkdown/container_block_processor.py": {
-            "too-many-arguments": 26,
+            "too-many-arguments": 25,
             "too-many-locals": 15,
             "chained-comparison": 1
         },
@@ -129,7 +129,7 @@
         "pymarkdown/version.py": {}
     },
     "disables-by-name": {
-        "too-many-arguments": 125,
+        "too-many-arguments": 124,
         "too-many-locals": 38,
         "too-few-public-methods": 2,
         "no-self-argument": 1,

diff --git a/publish/test-results.json b/publish/test-results.json
@@ -479,15 +479,15 @@
             "totalTests": 101,
             "failedTests": 0,
             "errorTests": 0,
-            "skippedTests": 1,
+            "skippedTests": 0,
             "elapsedTimeInMilliseconds": 0
         },
         {
             "name": "test.test_markdown_nested_three_block_unordered",
             "totalTests": 101,
             "failedTests": 0,
             "errorTests": 0,
-            "skippedTests": 1,
+            "skippedTests": 0,
             "elapsedTimeInMilliseconds": 0
         },
         {

diff --git a/pymarkdown/block_quote_processor.py b/pymarkdown/block_quote_processor.py
@@ -1582,6 +1582,87 @@ def __decrease_stack(
             else:
                 break  # pragma: no cover
 
+    @staticmethod
+    def __block_list_block_kludge(
+        parser_state,
+        position_marker,
+        stack_count,
+        block_quote_data,
+    ):
+        POGGER.debug("original_line_to_parse>>$", parser_state.original_line_to_parse)
+        POGGER.debug("text_to_parse>>$", position_marker.text_to_parse)
+        POGGER.debug("index_number>>$", position_marker.index_number)
+        POGGER.debug("index_indent>>$", position_marker.index_indent)
+
+        POGGER.debug("stack_count>>$", stack_count)
+        POGGER.debug("block_quote_data.stack_count>>$", block_quote_data.stack_count)
+        POGGER.debug(
+            "block_quote_data.current_count>>$", block_quote_data.current_count
+        )
+
+        # KLUDGE!
+        skip = False
+        if (
+            parser_state.original_line_to_parse == position_marker.text_to_parse
+            and not position_marker.index_indent
+        ):
+            last_active_block_quote_stack_index = position_marker.text_to_parse[
+                : position_marker.index_number + 1
+            ].count(">")
+            POGGER.debug(
+                "last_active_block_quote_stack_index>>$",
+                last_active_block_quote_stack_index,
+            )
+
+            text_after_current_block_quote = position_marker.text_to_parse[
+                position_marker.index_number + 1 :
+            ]
+            found_index = next(
+                (
+                    char_index
+                    for char_index, current_char in enumerate(
+                        text_after_current_block_quote
+                    )
+                    if current_char not in " >"
+                ),
+                -1,
+            )
+            POGGER.debug("found_index:$", found_index)
+            assert found_index < len(text_after_current_block_quote)
+            whitespace_after_block_quote = text_after_current_block_quote[:found_index]
+            text_after_block_quote = text_after_current_block_quote[found_index:]
+            POGGER.debug(
+                "whitespace_after_block_quote:$: + text_after_block_quote:$:",
+                whitespace_after_block_quote,
+                text_after_block_quote,
+            )
+
+            POGGER.debug("token_stack>>$", parser_state.token_stack)
+            POGGER.debug(
+                "token_stack[last_active_block_quote_stack_index]>>$",
+                parser_state.token_stack[last_active_block_quote_stack_index],
+            )
+            stack_index_valid = last_active_block_quote_stack_index + 1 < len(
+                parser_state.token_stack
+            )
+            stack_index_in_scope = last_active_block_quote_stack_index < stack_count
+            more_block_quotes_present = ">" in whitespace_after_block_quote
+            POGGER.debug(
+                "stack_index_valid:$ and stack_index_in_scope:$ and more_block_quotes_present:$",
+                stack_index_valid,
+                stack_index_in_scope,
+                more_block_quotes_present,
+            )
+            if stack_index_valid and stack_index_in_scope and more_block_quotes_present:
+                POGGER.debug(
+                    "xy>>$",
+                    parser_state.token_stack[last_active_block_quote_stack_index + 1],
+                )
+                skip = parser_state.token_stack[
+                    last_active_block_quote_stack_index + 1
+                ].is_list
+        return skip
+
     # pylint: disable=too-many-arguments
     @staticmethod
     def __increase_stack(
@@ -1632,12 +1713,19 @@ def __increase_stack(
             parser_state.token_stack.append(BlockQuoteStackToken(new_markdown_token))
 
         POGGER.debug("container_level_tokens>>$", container_level_tokens)
-        BlockQuoteProcessor.__decrease_stack_to_level(
+        skip = BlockQuoteProcessor.__block_list_block_kludge(
             parser_state,
-            block_quote_data.current_count,
+            position_marker,
             stack_count,
-            container_level_tokens,
+            block_quote_data,
         )
+        if not skip:
+            BlockQuoteProcessor.__decrease_stack_to_level(
+                parser_state,
+                block_quote_data.current_count,
+                stack_count,
+                container_level_tokens,
+            )
         POGGER.debug(
             "container_level_tokens>>$",
             container_level_tokens,

diff --git a/pymarkdown/container_block_processor.py b/pymarkdown/container_block_processor.py
@@ -139,7 +139,6 @@ def parse_line_for_container_blocks(
             and is_not_in_root_list
         ):
             POGGER.debug("indent")
-            did_blank, force_leaf_token_parse = False, False
             (
                 can_continue,
                 line_to_parse,
@@ -155,11 +154,12 @@ def parse_line_for_container_blocks(
                 skip_containers_before_leaf_blocks,
                 indent_already_processed,
                 was_other_paragraph_continuation,
+                did_blank,
+                force_leaf_token_parse,
             ) = ContainerBlockProcessor.__handle_indented_block_start(
                 parser_state, position_marker
             )
         else:
-            was_other_paragraph_continuation = False
             (
                 can_continue,
                 container_level_tokens,
@@ -178,6 +178,7 @@ def parse_line_for_container_blocks(
                 indent_already_processed,
                 force_leaf_token_parse,
                 did_blank,
+                was_other_paragraph_continuation,
             ) = ContainerBlockProcessor.__handle_non_leaf_block(
                 parser_state,
                 position_marker,
@@ -268,7 +269,8 @@ def __handle_non_leaf_block(
             last_list_start_index,
             did_blank,
             requeue_line_info,
-        ) = (False, None, None, None, None, None, None, None, False, None)
+        ) = (False, None, False, None, None, None, None, None, False, None)
+        was_other_paragraph_continuation = False
 
         if (
             parser_state.token_stack
@@ -381,6 +383,7 @@ def __handle_non_leaf_block(
             indent_already_processed,
             force_leaf_token_parse,
             did_blank,
+            was_other_paragraph_continuation,
         )
 
     # pylint: enable=too-many-arguments, too-many-locals
@@ -416,7 +419,7 @@ def __calculate_container_used_indent(
             stack_index += 1
         assert stack_index > container_depth
         _, extracted_whitespace = ParserHelper.extract_whitespace(
-            position_marker.text_to_parse, container_used_indent
+            parser_state.original_line_to_parse, container_used_indent
         )
         return container_used_indent, extracted_whitespace
 
@@ -511,6 +514,7 @@ def __determine_leading_whitespace_preprocessing(
                 "position_marker.index_number:$:", position_marker.index_number
             )
             POGGER.debug("container_depth:$:", container_depth)
+            POGGER.debug("extracted_whitespace:$:", extracted_whitespace)
             if position_marker.index_number == -1 and container_depth:
                 (
                     container_used_indent,
@@ -768,10 +772,10 @@ def __handle_leading_whitespace(
                 used_indent,
                 container_used_indent,
             )
-            # POGGER.debug(">do_break:$:", do_break)
-            # POGGER.debug(">stack_index:$:", stack_index)
-            # POGGER.debug(">remaining_whitespace:$:", remaining_whitespace)
-            # POGGER.debug(">used_indent:$:", used_indent)
+            POGGER.debug(">do_break:$:", do_break)
+            POGGER.debug(">stack_index:$:", stack_index)
+            POGGER.debug(">remaining_whitespace:$:", remaining_whitespace)
+            POGGER.debug(">used_indent:$:", used_indent)
             if do_break:
                 POGGER.debug(">break!")
                 break
@@ -831,6 +835,8 @@ def __handle_indented_block_start(parser_state, position_marker):
             last_list_start_index,
             indent_already_processed,
             was_other_paragraph_continuation,
+            did_blank,
+            force_leaf_token_parse,
         ) = (
             True,
             position_marker.text_to_parse,
@@ -845,6 +851,8 @@ def __handle_indented_block_start(parser_state, position_marker):
             -1,
             False,
             False,
+            False,
+            False,
         )
         POGGER.debug("was_paragraph_continuation>>$", was_paragraph_continuation)
         POGGER.debug("parser_state.token_stack>>$", parser_state.token_stack)
@@ -888,6 +896,8 @@ def __handle_indented_block_start(parser_state, position_marker):
             False,
             indent_already_processed,
             was_other_paragraph_continuation,
+            did_blank,
+            force_leaf_token_parse,
         )
 
     # pylint: enable=too-many-locals
@@ -2952,7 +2962,6 @@ def __adjust_containers_before_leaf_blocks(
 
     # pylint: enable=too-many-arguments
 
-    # pylint: disable=too-many-arguments
     @staticmethod
     def __handle_special_block_quote_reduction(
         parser_state,
@@ -2997,8 +3006,6 @@ def __handle_special_block_quote_reduction(
             close_tokens = []
         return close_tokens, None
 
-    # pylint: enable=too-many-arguments
-
     # pylint: disable=too-many-arguments, too-many-locals
     @staticmethod
     def __process_leaf_tokens(

diff --git a/test/test_markdown_nested_three_block_ordered.py b/test/test_markdown_nested_three_block_ordered.py
@@ -3228,8 +3228,7 @@ def test_nested_three_block_max_ordered_max_block_max():
 
 
 @pytest.mark.gfm
-@pytest.mark.skip
-def test_nested_three_block_max_ordered_max_block_max_empty():
+def test_nested_three_block_max_ordered_max_block_max_empty_x():
     """
     Verify that a nesting of block quote, ordered list, block quote, with
     the maximum number of spaces allowed, and no text on the first line, works properly.
@@ -3241,9 +3240,10 @@ def test_nested_three_block_max_ordered_max_block_max_empty():
     expected_tokens = [
         "[block-quote(1,4):   :   > ]",
         "[olist(1,9):.:1:14:   :]",
-        "[block-quote(1,15)::> \n   >          > ]",
-        "[para(1,17):\n]",
-        "[text(1,17):list\nitem::\n]",
+        "[block-quote(1,15)::>\n   >          > ]",
+        "[BLANK(1,16):]",
+        "[para(2,17):]",
+        "[text(2,17):item:]",
         "[end-para:::True]",
         "[end-block-quote:::True]",
         "[end-olist:::True]",
@@ -3260,7 +3260,7 @@ def test_nested_three_block_max_ordered_max_block_max_empty():
 </blockquote>"""
 
     # Act & Assert
-    act_and_assert(source_markdown, expected_gfm, expected_tokens)
+    act_and_assert(source_markdown, expected_gfm, expected_tokens, show_debug=False)
 
 
 @pytest.mark.gfm

diff --git a/test/test_markdown_nested_three_block_unordered.py b/test/test_markdown_nested_three_block_unordered.py
@@ -3228,7 +3228,6 @@ def test_nested_three_block_max_unordered_max_block_max():
 
 
 @pytest.mark.gfm
-@pytest.mark.skip
 def test_nested_three_block_max_unordered_max_block_max_empty():
     """
     Verify that a nesting of block quote, unordered list, block quote, with
@@ -3241,9 +3240,10 @@ def test_nested_three_block_max_unordered_max_block_max_empty():
     expected_tokens = [
         "[block-quote(1,4):   :   > ]",
         "[ulist(1,9):+::13:   :]",
-        "[block-quote(1,14)::> \n   >         > ]",
-        "[para(1,16):\n]",
-        "[text(1,16):list\nitem::\n]",
+        "[block-quote(1,14)::>\n   >         > ]",
+        "[BLANK(1,15):]",
+        "[para(2,16):]",
+        "[text(2,16):item:]",
         "[end-para:::True]",
         "[end-block-quote:::True]",
         "[end-ulist:::True]",
@@ -3408,7 +3408,7 @@ def test_nested_three_block_max_unordered_max_block_max_empty_no_bq2():
 </blockquote>"""
 
     # Act & Assert
-    act_and_assert(source_markdown, expected_gfm, expected_tokens)
+    act_and_assert(source_markdown, expected_gfm, expected_tokens, show_debug=True)
 
 
 @pytest.mark.gfm