Skip to content

Commit

Permalink
https://github.com/jackdewinter/pymarkdown/issues/836
Browse files Browse the repository at this point in the history
  • Loading branch information
@jackdewinter
jackdewinter committed Nov 18, 2023
1 parent b729dde commit 2af6a3f
Show file tree
Hide file tree
Showing 3 changed files with 107 additions and 130 deletions.
90 changes: 74 additions & 16 deletions pymarkdown/leaf_blocks/fenced_leaf_block_processor.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -198,11 +198,29 @@ def __check_for_fenced_end(
extracted_whitespace,
split_tab,
) = FencedLeafBlockProcessor.__check_for_fenced_end_with_tab(
parser_state,
position_marker,
original_line,
detabified_original_start_index,
collected_count,
extracted_whitespace,
)
if split_tab:
reconstructed_line = position_marker.text_to_parse
was_indented = not parser_state.token_stack[-2].is_document
pp = " " * position_marker.index_indent
my_abc = False
if original_line.startswith(">"):
my_abc = True
pp = None
(
adj_original,
adj_original_index,
split_tab,
) = TabHelper.find_tabified_string(
original_line, reconstructed_line, abc=my_abc, was_indented=was_indented, reconstruct_prefix=pp
)
my_ws = original_line[:adj_original_index]

after_fence_and_spaces_index, extracted_spaces = ParserHelper.extract_spaces(
position_marker.text_to_parse, after_fence_index
Expand All @@ -226,8 +244,10 @@ def __check_for_fenced_end(
and after_fence_and_spaces_index >= len(position_marker.text_to_parse)
and only_spaces_after_fence
):
was_indented = not parser_state.token_stack[-2].is_document
if split_tab:
TabHelper.adjust_block_quote_indent_for_tab(parser_state)
# my_ws = None
TabHelper.adjust_block_quote_indent_for_tab(parser_state, my_ws)

assert extracted_whitespace is not None
assert extracted_spaces is not None
Expand All @@ -245,6 +265,8 @@ def __check_for_fenced_end(

@staticmethod
def __check_for_fenced_end_with_tab(
parser_state: ParserState,
position_marker: PositionMarker,
original_line: str,
detabified_original_start_index: int,
collected_count: int,
Expand Down Expand Up @@ -273,8 +295,15 @@ def __check_for_fenced_end_with_tab(

assert extracted_whitespace is not None

was_indented = not parser_state.token_stack[-2].is_document
pp = " " * position_marker.index_indent
my_abc = False
if original_line.startswith(">"):
my_abc = True
pp = None

_, adj_original_index, split_tab = TabHelper.find_tabified_string(
original_line, extracted_whitespace + adj_end
original_line, extracted_whitespace + adj_end, was_indented = was_indented, reconstruct_prefix=pp
)

_, new_extracted_whitespace = ParserHelper.extract_spaces(
Expand Down Expand Up @@ -488,6 +517,7 @@ def __add_fenced_tokens(
corrected_prefix_length,
extracted_text,
text_after_extracted_text,
my_ws
) = FencedLeafBlockProcessor.__add_fenced_tokens_prepare(
position_marker,
original_line,
Expand All @@ -510,11 +540,12 @@ def __add_fenced_tokens(
line_to_parse,
split_tab,
block_quote_data,
my_ws
)

@staticmethod
def __add_fenced_tokens_calc(
parser_state: ParserState, split_tab: bool, block_quote_data: BlockQuoteData
parser_state: ParserState, split_tab: bool, block_quote_data: BlockQuoteData, my_ws:Optional[str]
) -> Tuple[StackToken, List[MarkdownToken], int]:
old_top_of_stack = parser_state.token_stack[-1]
new_tokens, _ = parser_state.close_open_blocks_fn(
Expand All @@ -525,7 +556,7 @@ def __add_fenced_tokens_calc(
if split_tab := ContainerHelper.reduce_containers_if_required(
parser_state, block_quote_data, new_tokens, split_tab
):
TabHelper.adjust_block_quote_indent_for_tab(parser_state)
TabHelper.adjust_block_quote_indent_for_tab(parser_state, extracted_whitespace=my_ws)
whitespace_count_delta = -1
else:
whitespace_count_delta = 0
Expand All @@ -541,14 +572,15 @@ def __add_fenced_tokens_prepare(
collected_count: int,
extracted_whitespace: Optional[str],
after_fence_index: int,
) -> Tuple[str, int, Optional[str], Optional[str], bool, int, str, str]:
) -> Tuple[str, int, Optional[str], Optional[str], bool, int, str, str, Optional[str]]:
(
line_to_parse,
non_whitespace_index,
extracted_whitespace_before_info_string,
extracted_whitespace,
split_tab,
corrected_prefix_length,
my_ws
) = FencedLeafBlockProcessor.__add_fenced_tokens_with_tab(
position_marker,
original_line,
Expand Down Expand Up @@ -583,6 +615,7 @@ def __add_fenced_tokens_prepare(
corrected_prefix_length,
extracted_text,
line_to_parse[after_extracted_text_index:],
my_ws
)

# pylint: enable=too-many-arguments
Expand All @@ -600,13 +633,14 @@ def __add_fenced_tokens_create(
line_to_parse: str,
split_tab: bool,
block_quote_data: BlockQuoteData,
my_ws:Optional[str]
) -> Tuple[StackToken, List[MarkdownToken]]:
(
old_top_of_stack,
new_tokens,
whitespace_start_count,
) = FencedLeafBlockProcessor.__add_fenced_tokens_calc(
parser_state, split_tab, block_quote_data
parser_state, split_tab, block_quote_data, my_ws
)

pre_extracted_text, pre_text_after_extracted_text = (
Expand Down Expand Up @@ -640,9 +674,14 @@ def __add_fenced_tokens_create(
)
new_tokens.append(new_token)
assert extracted_whitespace is not None
whitespace_start_count += TabHelper.calculate_length(
extracted_whitespace, corrected_prefix_length
)
if my_ws is not None:
whitespace_start_count += TabHelper.calculate_length(
my_ws, 0
)
else:
whitespace_start_count += TabHelper.calculate_length(
extracted_whitespace, corrected_prefix_length
)
parser_state.token_stack.append(
FencedCodeBlockStackToken(
code_fence_character=line_to_parse[position_marker.index_number],
Expand All @@ -665,17 +704,19 @@ def __add_fenced_tokens_with_tab(
collected_count: int,
extracted_whitespace: Optional[str],
after_fence_index: int,
) -> Tuple[str, int, Optional[str], Optional[str], bool, int]:
) -> Tuple[str, int, Optional[str], Optional[str], bool, int, Optional[str]]:
split_tab = False
corrected_prefix_length = 0
line_to_parse = position_marker.text_to_parse
my_ws:Optional[str] = None
if ParserHelper.tab_character in original_line:
(
fence_string,
adj_original_line,
split_tab,
extracted_whitespace,
corrected_prefix_length,
my_ws
) = FencedLeafBlockProcessor.__add_fenced_tokens_with_tab_calc(
original_line,
line_to_parse,
Expand Down Expand Up @@ -709,6 +750,7 @@ def __add_fenced_tokens_with_tab(
extracted_whitespace,
split_tab,
corrected_prefix_length,
my_ws
)

# pylint: enable=too-many-arguments
Expand All @@ -722,10 +764,11 @@ def __add_fenced_tokens_with_tab_calc(
collected_count: int,
after_fence_index: int,
extracted_whitespace: Optional[str],
) -> Tuple[str, str, bool, Optional[str], int]:
) -> Tuple[str, str, bool, Optional[str], int, Optional[str]]:
fence_string = line_to_parse[new_index - collected_count : new_index]
split_tab = False
corrected_prefix_length = 0
my_ws:Optional[str] = None

adj_original_line, _, _ = TabHelper.find_detabify_string(
original_line, line_to_parse, use_proper_traverse=True
Expand Down Expand Up @@ -755,13 +798,15 @@ def __add_fenced_tokens_with_tab_calc(
extracted_whitespace = corrected_suffix
corrected_prefix_length = len(corrected_prefix)
if split_tab:
assert split_tab_with_block_quote_suffix
split_tab_with_block_quote_suffix = None
my_ws =corrected_prefix+corrected_suffix
return (
fence_string,
adj_original_line,
split_tab,
extracted_whitespace,
corrected_prefix_length,
my_ws
)

# pylint: enable=too-many-arguments
Expand Down Expand Up @@ -823,7 +868,7 @@ def handle_fenced_code_block(
leaf_token_whitespace,
token_text,
) = FencedLeafBlockProcessor.__handle_fenced_code_block_with_tab(
parser_state, original_line, leaf_token_whitespace, token_text
parser_state, position_marker, original_line, leaf_token_whitespace, token_text
)
new_tokens.append(
TextMarkdownToken(
Expand Down Expand Up @@ -895,6 +940,7 @@ def __handle_fenced_code_block_with_tab_and_extracted_whitespace(
@staticmethod
def __handle_fenced_code_block_with_tab(
parser_state: ParserState,
position_marker: PositionMarker,
original_line: str,
leaf_token_whitespace: str,
token_text: str,
Expand All @@ -910,6 +956,7 @@ def __handle_fenced_code_block_with_tab(
)
reconstructed_line = resolved_leaf_token_whitespace + token_text
reconstructed_line_has_tab = True
my_ws : Optional[str] = None
if reconstructed_line[0] == "\t":
reconstructed_line_has_tab = False
adj_original = reconstructed_line
Expand All @@ -921,13 +968,22 @@ def __handle_fenced_code_block_with_tab(
resolved_leaf_token_whitespace, adj_original_index
)
else:
pp = " " * position_marker.index_indent
my_abc = False
if original_line.startswith(">"):
my_abc = True
pp = None
(
adj_original,
adj_original_index,
split_tab,
) = TabHelper.find_tabified_string(
original_line, reconstructed_line, abc=True, was_indented=was_indented
original_line, reconstructed_line, abc=my_abc, was_indented=was_indented, reconstruct_prefix=pp
)
if split_tab:
my_ws = original_line[:adj_original_index]
if not my_abc:
adj_original_index += position_marker.index_indent

(
leaf_token_whitespace,
Expand All @@ -942,6 +998,7 @@ def __handle_fenced_code_block_with_tab(
was_indented,
reconstructed_line_has_tab,
split_tab,
my_ws
)
return leaf_token_whitespace, token_text

Expand All @@ -957,6 +1014,7 @@ def __handle_fenced_code_block_with_tab_whitespace(
was_indented: bool,
reconstructed_line_has_tab: bool,
split_tab: bool,
my_ws:Optional[str]
) -> Tuple[str, str]:
space_end_index, extracted_whitespace = ParserHelper.extract_spaces(
adj_original, 0
Expand All @@ -967,7 +1025,7 @@ def __handle_fenced_code_block_with_tab_whitespace(
detabified_extracted_whitespace = TabHelper.detabify_string(
extracted_whitespace, adj_original_index
)
assert detabified_extracted_whitespace == resolved_leaf_token_whitespace
#assert detabified_extracted_whitespace == resolved_leaf_token_whitespace

new_extracted_whitespace = extracted_whitespace
if new_extracted_whitespace and whitespace_start_count:
Expand All @@ -983,7 +1041,7 @@ def __handle_fenced_code_block_with_tab_whitespace(
leaf_token_whitespace = new_extracted_whitespace
token_text = adj_original[space_end_index:]
if split_tab:
TabHelper.adjust_block_quote_indent_for_tab(parser_state)
TabHelper.adjust_block_quote_indent_for_tab(parser_state, my_ws)

return leaf_token_whitespace, token_text

Expand Down
12 changes: 6 additions & 6 deletions pymarkdown/tokens/list_start_markdown_token.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -163,12 +163,12 @@ def remove_last_leading_space(self) -> Optional[str]:
"""
assert self.__leading_spaces is not None
last_separator_index = self.__leading_spaces.rfind("\n")
assert last_separator_index == -1
extracted_text = self.__leading_spaces
self.__leading_spaces = None
# else:
# extracted_text = self.__leading_spaces[last_separator_index:]
# self.__leading_spaces = self.__leading_spaces[:last_separator_index]
if last_separator_index == -1:
extracted_text = self.__leading_spaces
self.__leading_spaces = None
else:
extracted_text = self.__leading_spaces[last_separator_index:]
self.__leading_spaces = self.__leading_spaces[:last_separator_index]
self.__compose_extra_data_field()
return extracted_text

Expand Down
Loading

0 comments on commit 2af6a3f

Please sign in to comment.