Issue 1007 (#1020)

* updating

* upgrade

* #1007

publish/coverage.json

@@ -2,12 +2,12 @@
     "projectName": "pymarkdown",
     "reportSource": "pytest",
     "branchLevel": {
-        "totalMeasured": 4839,
-        "totalCovered": 4839
+        "totalMeasured": 4889,
+        "totalCovered": 4889
     },
     "lineLevel": {
-        "totalMeasured": 19456,
-        "totalCovered": 19456
+        "totalMeasured": 19585,
+        "totalCovered": 19585
     }
 }
 

publish/pylint_suppression.json

@@ -145,7 +145,7 @@
         "pymarkdown/inline/inline_line_end_helper.py": {
             "too-few-public-methods": 1,
             "too-many-arguments": 4,
-            "too-many-locals": 1
+            "too-many-locals": 2
         },
         "pymarkdown/inline/inline_processor.py": {},
         "pymarkdown/inline/inline_request.py": {
@@ -155,7 +155,8 @@
             "too-many-instance-attributes": 1
         },
         "pymarkdown/inline/inline_tabified_text_block_helper.py": {
-            "too-many-arguments": 1
+            "too-many-arguments": 2,
+            "too-many-locals": 1
         },
         "pymarkdown/inline/inline_text_block_helper.py": {
             "too-few-public-methods": 1,
@@ -164,7 +165,7 @@
         },
         "pymarkdown/leaf_blocks/atx_leaf_block_processor.py": {
             "too-many-arguments": 5,
-            "too-many-locals": 2
+            "too-many-locals": 3
         },
         "pymarkdown/leaf_blocks/fenced_leaf_block_processor.py": {
             "too-many-arguments": 12,
@@ -302,7 +303,9 @@
         },
         "pymarkdown/plugins/rule_md_021.py": {},
         "pymarkdown/plugins/rule_md_022.py": {},
-        "pymarkdown/plugins/rule_md_023.py": {},
+        "pymarkdown/plugins/rule_md_023.py": {
+            "too-many-arguments": 1
+        },
         "pymarkdown/plugins/rule_md_024.py": {},
         "pymarkdown/plugins/rule_md_025.py": {},
         "pymarkdown/plugins/rule_md_026.py": {},
@@ -497,8 +500,8 @@
         "too-many-instance-attributes": 24,
         "too-many-public-methods": 4,
         "too-few-public-methods": 39,
-        "too-many-arguments": 228,
-        "too-many-locals": 38,
+        "too-many-arguments": 230,
+        "too-many-locals": 41,
         "chained-comparison": 1,
         "too-many-boolean-expressions": 2,
         "protected-access": 26,

publish/test-results.json

@@ -194,6 +194,14 @@
             "skippedTests": 0,
             "elapsedTimeInMilliseconds": 0
         },
+        {
+            "name": "test.basic.test_get_replacement_indices",
+            "totalTests": 4,
+            "failedTests": 0,
+            "errorTests": 0,
+            "skippedTests": 0,
+            "elapsedTimeInMilliseconds": 0
+        },
         {
             "name": "test.basic.test_html_tags",
             "totalTests": 29,
@@ -1252,7 +1260,7 @@
         },
         {
             "name": "test.rules.test_md023",
-            "totalTests": 31,
+            "totalTests": 106,
             "failedTests": 0,
             "errorTests": 0,
             "skippedTests": 0,
@@ -1604,10 +1612,10 @@
         },
         {
             "name": "test.test_markdown_extra",
-            "totalTests": 115,
+            "totalTests": 133,
             "failedTests": 0,
             "errorTests": 0,
-            "skippedTests": 0,
+            "skippedTests": 1,
             "elapsedTimeInMilliseconds": 0
         },
         {

pymarkdown/general/parser_helper.py

@@ -696,6 +696,41 @@ def __remove_escapes_from_text(token_text: str) -> str:
         """
         return ParserHelper.__resolve_escapes_from_text(token_text)
 
+    @staticmethod
+    def is_character_replacement_marker(
+        source_string: str, index_in_string: int
+    ) -> bool:
+        """
+        Determine if the specified character is a replacement marker character.
+        """
+        return (
+            0 <= index_in_string < len(source_string)
+            and source_string[index_in_string] == ParserHelper.__alert_character
+        )
+
+    @staticmethod
+    def get_replacement_indices(
+        source_string: str, index_in_string: int
+    ) -> Tuple[int, int, int]:
+        """
+        Find the next replacement character index, as well as the middle and end indices
+        for the replacement.
+        """
+        start_replacement_index = ParserHelper.__find_with_escape(
+            source_string, ParserHelper.__alert_character, index_in_string
+        )
+        middle_replacement_index = -1
+        end_replacement_index = -1
+        if start_replacement_index != -1:
+            middle_replacement_index = source_string.find(
+                ParserHelper.__alert_character, start_replacement_index + 1
+            )
+            if middle_replacement_index != -1:
+                end_replacement_index = source_string.find(
+                    ParserHelper.__alert_character, middle_replacement_index + 1
+                )
+        return start_replacement_index, middle_replacement_index, end_replacement_index
+
     @staticmethod
     def __resolve_replacement_markers_from_text(main_text: str) -> str:
         """

pymarkdown/inline/inline_backtick_helper.py

@@ -287,7 +287,6 @@ def __calculate_backtick_between_tabified_text(
             inline_request.tabified_text
         )
         POGGER.debug("split_tabified_lines>>$<<", split_tabified_lines)
-        assert len(split_tabified_lines) == len(split_source_lines)
 
         (
             start_array_index,

pymarkdown/inline/inline_line_end_helper.py

@@ -39,7 +39,9 @@ def __handle_line_end(
         coalesced_stack: List[MarkdownToken],
         tabified_text: Optional[str],
         inline_request: InlineRequest,
-    ) -> Tuple[str, Optional[str], List[MarkdownToken], str, Optional[str], str]:
+    ) -> Tuple[
+        str, Optional[str], List[MarkdownToken], str, Optional[str], str, Optional[str]
+    ]:
         """
         Handle the inline case of having the end of line character encountered.
         """
@@ -64,6 +66,7 @@ def __handle_line_end(
             append_to_current_string,
             end_string,
             remaining_line,
+            tabified_remaining_line,
         ) = InlineLineEndHelper.__select_line_ending(
             new_tokens,
             line_number,
@@ -76,6 +79,7 @@ def __handle_line_end(
             is_setext,
             tabified_text,
             inline_request,
+            tabified_remaining_line,
         )
 
         InlineLineEndHelper.__handle_line_end_adjust_block_quote(coalesced_stack)
@@ -87,6 +91,7 @@ def __handle_line_end(
             remaining_line,
             end_string,
             current_string,
+            tabified_remaining_line,
         )
 
     # pylint: enable=too-many-arguments
@@ -138,7 +143,26 @@ def __is_proper_hard_break(
         POGGER.debug("__is_proper_hard_break>>$>>", is_proper_hard_break)
         return is_proper_hard_break
 
-    # pylint: disable=too-many-arguments
+    @staticmethod
+    def __select_line_end_hard_break(
+        new_tokens: List[MarkdownToken],
+        line_number: int,
+        adj_hard_column: int,
+        current_string: str,
+    ) -> Tuple[str, Optional[str], str]:
+        POGGER.debug(">>proper hard break")
+        new_tokens.append(
+            HardBreakMarkdownToken(
+                InlineBackslashHelper.backslash_character,
+                line_number,
+                adj_hard_column - 1,
+            )
+        )
+        current_string, whitespace_to_add = current_string[:-1], None
+        append_to_current_string = ""
+        return current_string, whitespace_to_add, append_to_current_string
+
+    # pylint: disable=too-many-arguments, too-many-locals
     @staticmethod
     def __select_line_ending(
         new_tokens: List[MarkdownToken],
@@ -152,7 +176,8 @@ def __select_line_ending(
         is_setext: bool,
         tabified_text: Optional[str],
         inline_request: InlineRequest,
-    ) -> Tuple[str, Optional[str], str, Optional[str], str]:
+        tabified_remaining_line: Optional[str],
+    ) -> Tuple[str, Optional[str], str, Optional[str], str, Optional[str]]:
         # POGGER.debug(">>removed_end_whitespace>:$:<", removed_end_whitespace)
         # POGGER.debug(">>tabified_text>:$:<", tabified_text)
         # POGGER.debug(
@@ -172,21 +197,18 @@ def __select_line_ending(
         #     remaining_line,
         # )
 
-        is_proper_end = not tabified_text or tabified_text.endswith("  ")
+        is_proper_end = not tabified_text or (
+            tabified_remaining_line and tabified_remaining_line.endswith("  ")
+        )
 
         if InlineLineEndHelper.__is_proper_hard_break(
             current_string, removed_end_whitespace_size
         ):
-            POGGER.debug(">>proper hard break")
-            new_tokens.append(
-                HardBreakMarkdownToken(
-                    InlineBackslashHelper.backslash_character,
-                    line_number,
-                    adj_hard_column - 1,
+            current_string, whitespace_to_add, append_to_current_string = (
+                InlineLineEndHelper.__select_line_end_hard_break(
+                    new_tokens, line_number, adj_hard_column, current_string
                 )
             )
-            current_string, whitespace_to_add = current_string[:-1], None
-            append_to_current_string = ""
         elif removed_end_whitespace_size >= 2 and is_proper_end:
             POGGER.debug(">>whitespace hard break")
             new_tokens.append(
@@ -196,6 +218,18 @@ def __select_line_ending(
             )
             whitespace_to_add = None
             append_to_current_string = ""
+
+            if tabified_remaining_line is not None:
+                number_collected_characters, start_index = (
+                    ParserHelper.collect_backwards_while_character(
+                        tabified_remaining_line, len(tabified_remaining_line), " "
+                    )
+                )
+                assert (
+                    number_collected_characters is not None
+                    and number_collected_characters >= 2
+                )
+                tabified_remaining_line = tabified_remaining_line[:start_index]
         else:
             POGGER.debug(">>normal end")
             # POGGER.debug("current_string>:$:<", current_string)
@@ -232,9 +266,10 @@ def __select_line_ending(
             append_to_current_string,
             end_string,
             remaining_line,
+            tabified_remaining_line,
         )
 
-    # pylint: enable=too-many-arguments
+    # pylint: enable=too-many-arguments, too-many-locals
 
     # pylint: disable=too-many-arguments
     @staticmethod
@@ -262,8 +297,9 @@ def __select_line_ending_normal(
             new_index, ex_ws = ParserHelper.extract_spaces(remaining_line, 0)
             # POGGER.debug("<<new_index<<$<<", new_index)
             # POGGER.debug("<<ex_ws<<$<<", ex_ws)
-            assert new_index
-            end_string = f"{ex_ws}{ParserHelper.whitespace_split_character}"
+            end_string = (
+                f"{ex_ws}{ParserHelper.whitespace_split_character}" if new_index else ""
+            )
             remaining_line = remaining_line[new_index:]
         if not is_setext and tabified_remaining_line and removed_end_whitespace:
             POGGER.debug("<<tabified_remaining_line>:$:<", tabified_remaining_line)
@@ -325,6 +361,7 @@ def process_inline_new_line(
             remaining_line,
             end_string,
             current_string,
+            tabified_remaining_line,
         ) = InlineLineEndHelper.__handle_line_end(
             remaining_line,
             tabified_remaining_line,
@@ -408,8 +445,8 @@ def __clean_up_new_line(
         newline_index = end_suffix.rfind("\n")
         if is_setext:
             special_index = end_suffix.rfind(ParserHelper.whitespace_split_character)
-            if special_index != -1 and newline_index != 1:
-                max_index = max(special_index, special_index)
+            if special_index != -1 or newline_index != -1:
+                max_index = max(special_index, newline_index)
                 end_suffix = end_suffix[max_index + 1 :]
             else:
                 assert special_index == -1