Merge pull request #1547 from lark-parser/dev

Bugfix in indenter - now always creating dedents with line information

Author: erezsh

lark/exceptions.py

@@ -60,8 +60,7 @@ def get_context(self, text: str, span: int=40) -> str:
             The parser doesn't hold a copy of the text it has to parse,
             so you have to provide it again
         """
-        assert self.pos_in_stream is not None, self
-        pos = self.pos_in_stream
+        pos = self.pos_in_stream or 0
         start = max(pos - span, 0)
         end = pos + span
         if not isinstance(text, bytes):

lark/indenter.py

@@ -55,6 +55,7 @@ def handle_NL(self, token: Token) -> Iterator[Token]:
                 raise DedentError('Unexpected dedent to column %s. Expected dedent to %s' % (indent, self.indent_level[-1]))
 
     def _process(self, stream):
+        token = None
         for token in stream:
             if token.type == self.NL_type:
                 yield from self.handle_NL(token)
@@ -69,7 +70,7 @@ def _process(self, stream):
 
         while len(self.indent_level) > 1:
             self.indent_level.pop()
-            yield Token(self.DEDENT_type, '')
+            yield Token.new_borrow_pos(self.DEDENT_type, '', token) if token else Token(self.DEDENT_type, '', 0, 0, 0, 0, 0, 0)
 
         assert self.indent_level == [0], self.indent_level