From 00d83f911fc072d7e8590d86775ffe26e6e497b8 Mon Sep 17 00:00:00 2001 From: Chanic Panic <51764816+chanicpanic@users.noreply.github.com> Date: Tue, 1 Aug 2023 12:09:16 -0700 Subject: [PATCH 1/3] Add tests for issue #1283 --- tests/test_parser.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/test_parser.py b/tests/test_parser.py index ee5d5a07..13124a0d 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -954,6 +954,26 @@ def test_symbol_node_start_end_dynamic_lexer(self): self.assertEqual(node.start, 0) self.assertEqual(node.end, 3) + def test_resolve_ambiguity_with_shared_node(self): + grammar = """ + start: (a+)* + !a.1: "A" | + """ + + l = Lark(grammar, ambiguity='resolve', lexer=LEXER) + tree = l.parse("A") + self.assertEqual(tree, Tree('start', [Tree('a', []), Tree('a', []), Tree('a', ['A'])])) + + def test_resolve_ambiguity_with_shared_node2(self): + grammar = """ + start: _s x _s + x: "X"? + _s: " "? + """ + + l = Lark(grammar, ambiguity='resolve', lexer=LEXER) + tree = l.parse("") + self.assertEqual(tree, Tree('start', [Tree('x', [])])) _NAME = "TestFullEarley" + LEXER.capitalize() _TestFullEarley.__name__ = _NAME From 50a8a143b07e7d571185a999ff4f99e0dac0a213 Mon Sep 17 00:00:00 2001 From: Chanic Panic <51764816+chanicpanic@users.noreply.github.com> Date: Tue, 1 Aug 2023 12:28:13 -0700 Subject: [PATCH 2/3] Disable ForestToParseTree cache when ambiguity='resolve' (issue #1283) --- lark/parsers/earley.py | 2 +- lark/parsers/earley_forest.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/lark/parsers/earley.py b/lark/parsers/earley.py index 2153a0ce..731c10ae 100644 --- a/lark/parsers/earley.py +++ b/lark/parsers/earley.py @@ -300,7 +300,7 @@ def parse(self, lexer, start): if self.Tree is not None: # Perform our SPPF -> AST conversion - transformer = ForestToParseTree(self.Tree, self.callbacks, self.forest_sum_visitor and self.forest_sum_visitor(), self.resolve_ambiguity) + transformer = ForestToParseTree(self.Tree, self.callbacks, self.forest_sum_visitor and self.forest_sum_visitor(), self.resolve_ambiguity, not self.resolve_ambiguity) solutions = [transformer.transform(s) for s in solutions] if len(solutions) > 1: diff --git a/lark/parsers/earley_forest.py b/lark/parsers/earley_forest.py index cdc613ab..c60f3a6b 100644 --- a/lark/parsers/earley_forest.py +++ b/lark/parsers/earley_forest.py @@ -609,9 +609,10 @@ def transform_packed_node(self, node, data): children.append(data.left) if data.right is not PackedData.NO_DATA: children.append(data.right) - if node.parent.is_intermediate: - return self._cache.setdefault(id(node), children) - return self._cache.setdefault(id(node), self._call_rule_func(node, children)) + transformed = children if node.parent.is_intermediate else self._call_rule_func(node, children) + if self._use_cache: + self._cache[id(node)] = transformed + return transformed def visit_symbol_node_in(self, node): super(ForestToParseTree, self).visit_symbol_node_in(node) From 4c1507aa5cb629e63058039158e564a28b965e7e Mon Sep 17 00:00:00 2001 From: Chanic Panic <51764816+chanicpanic@users.noreply.github.com> Date: Fri, 28 Jun 2024 18:40:28 -0700 Subject: [PATCH 3/3] Add comment explaining the change --- lark/parsers/earley.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lark/parsers/earley.py b/lark/parsers/earley.py index 731c10ae..8deaa2f4 100644 --- a/lark/parsers/earley.py +++ b/lark/parsers/earley.py @@ -300,7 +300,10 @@ def parse(self, lexer, start): if self.Tree is not None: # Perform our SPPF -> AST conversion - transformer = ForestToParseTree(self.Tree, self.callbacks, self.forest_sum_visitor and self.forest_sum_visitor(), self.resolve_ambiguity, not self.resolve_ambiguity) + # Disable the ForestToParseTree cache when ambiguity='resolve' + # to prevent a tree construction bug. See issue #1283 + use_cache = not self.resolve_ambiguity + transformer = ForestToParseTree(self.Tree, self.callbacks, self.forest_sum_visitor and self.forest_sum_visitor(), self.resolve_ambiguity, use_cache) solutions = [transformer.transform(s) for s in solutions] if len(solutions) > 1: