diff --git a/.github/workflows/consistency-checks.yml b/.github/workflows/consistency-checks.yml index f3fdc5d19..277f9d905 100644 --- a/.github/workflows/consistency-checks.yml +++ b/.github/workflows/consistency-checks.yml @@ -23,7 +23,7 @@ jobs: sudo apt update -qq && sudo apt install llvm-dev remake python -m pip install --upgrade pip # We can comment out after next Mathics-Scanner release - python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full] + python -m pip install -e git+https://github.com/Mathics3/mathics-scanner@position-tracking#egg=Mathics-Scanner[full] pip install -e . - name: Install Mathics with minimum dependencies diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 0432c6372..b84d1c493 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -35,7 +35,7 @@ jobs: cd .. # We can comment out after next Mathics-Scanner release # python -m pip install Mathics-Scanner[full] - python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full] + python -m pip install -e git+https://github.com/Mathics3/mathics-scanner@position-tracking#egg=Mathics-Scanner[full] pip install -e . remake -x develop-full - name: Test Mathics3 diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 5048ded14..c265453c3 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -27,7 +27,7 @@ jobs: run: | pip install mypy==1.13 sympy==1.12 # Adjust below for right branch - git clone --depth 1 https://github.com/Mathics3/mathics-scanner.git + git clone --depth 1 --branch position-tracking https://github.com/Mathics3/mathics-scanner cd mathics-scanner/ pip install -e . bash ./admin-tools/make-JSON-tables.sh diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 44c72f6b5..9efb8b13b 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -25,7 +25,7 @@ jobs: run: | python -m pip install --upgrade pip # We can comment out after next Mathics-Scanner release - python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full] + python -m pip install -e git+https://github.com/Mathics3/mathics-scanner@position-tracking#egg=Mathics-Scanner[full] - name: Run Mathics3 Combinatorica tests run: | git submodule init diff --git a/.github/workflows/pyodide.yml b/.github/workflows/pyodide.yml index 2780cf7b1..06c449e99 100644 --- a/.github/workflows/pyodide.yml +++ b/.github/workflows/pyodide.yml @@ -55,7 +55,7 @@ jobs: pip install "setuptools>=70.0.0" PyYAML click packaging pytest # We can comment out after next Mathics-Scanner release - python -m pip install --no-build-isolation -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner + python -m pip install --no-build-isolation -e git+https://github.com/Mathics3/mathics-scanner@position-tracking#egg=Mathics-Scanner # pip install --no-build-isolation -e . # cd .. diff --git a/.github/workflows/ubuntu-cython.yml b/.github/workflows/ubuntu-cython.yml index c57bba501..f81fa648b 100644 --- a/.github/workflows/ubuntu-cython.yml +++ b/.github/workflows/ubuntu-cython.yml @@ -30,7 +30,7 @@ jobs: pip install -e . cd .. # We can comment out after next Mathics-Scanner release - python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full] + python -m pip install -e git+https://github.com/Mathics3/mathics-scanner@position-tracking#egg=Mathics-Scanner[full] pip install -e . cd .. diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index ce76223be..2ea9e2d59 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -31,7 +31,7 @@ jobs: cd .. # We can comment out after next Mathics-Scanner release # python -m pip install Mathics-Scanner[full] - python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full] + python -m pip install -e git+https://github.com/Mathics3/mathics-scanner@position-tracking#egg=Mathics-Scanner[full] pip install -e . remake -x develop-full - name: Test Mathics diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index a112eba9c..abd233f8b 100755 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -39,7 +39,7 @@ jobs: pip install -e . cd .. # We can comment out after next Mathics-Scanner release - python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full] + python -m pip install -e git+https://github.com/Mathics3/mathics-scanner@position-tracking#egg=Mathics-Scanner[full] pip install -e . # python -m pip install Mathics-Scanner[full] diff --git a/examples/symbolic_logic/gries_schneider/test_gs.py b/examples/symbolic_logic/gries_schneider/test_gs.py index 19cd96d03..f2c74fc68 100644 --- a/examples/symbolic_logic/gries_schneider/test_gs.py +++ b/examples/symbolic_logic/gries_schneider/test_gs.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- +from mathics_scanner.location import ContainerKind from mathics.core.definitions import Definitions from mathics.core.evaluation import Evaluation @@ -13,5 +14,8 @@ for i in range(0, 4): evaluation = Evaluation(definitions=definitions, catch_interrupt=False) - expr = parse(definitions, MathicsSingleLineFeeder(f"<< GS{i}.m")) + expr = parse( + definitions, + MathicsSingleLineFeeder(f"<< GS{i}.m", "", ContainerKind.STRING), + ) expr.evaluate(evaluation) diff --git a/mathics/builtin/testing_expressions/string_tests.py b/mathics/builtin/testing_expressions/string_tests.py index bc800352a..aee037356 100644 --- a/mathics/builtin/testing_expressions/string_tests.py +++ b/mathics/builtin/testing_expressions/string_tests.py @@ -5,6 +5,7 @@ import re from mathics_scanner import SingleLineFeeder, SyntaxError +from mathics_scanner.location import ContainerKind from mathics.builtin.atomic.strings import anchor_pattern from mathics.core.atoms import Integer1, String @@ -278,7 +279,7 @@ def eval(self, string, evaluation: Evaluation): ) return - feeder = SingleLineFeeder(string.value) + feeder = SingleLineFeeder(string.value, "", ContainerKind.STRING) try: parser.parse(feeder) except SyntaxError: diff --git a/mathics/builtin/trace.py b/mathics/builtin/trace.py index d982c96fe..f8b3954a3 100644 --- a/mathics/builtin/trace.py +++ b/mathics/builtin/trace.py @@ -22,16 +22,24 @@ from time import time from typing import Callable +import mathics_scanner.location + import mathics.eval.tracing from mathics.core.attributes import A_HOLD_ALL, A_HOLD_ALL_COMPLETE, A_PROTECTED -from mathics.core.builtin import Builtin +from mathics.core.builtin import Builtin, Predefined from mathics.core.convert.python import from_bool, from_python from mathics.core.definitions import Definitions from mathics.core.evaluation import Evaluation from mathics.core.expression import Expression from mathics.core.list import ListExpression from mathics.core.rules import FunctionApplyRule -from mathics.core.symbols import SymbolFalse, SymbolNull, SymbolTrue, strip_context +from mathics.core.symbols import ( + Symbol, + SymbolFalse, + SymbolNull, + SymbolTrue, + strip_context, +) def traced_apply_function( @@ -493,3 +501,33 @@ def eval(self, expr: Expression, evaluation: Evaluation): else: result = expr.evaluate(evaluation) return ListExpression(result, profile_result) + + +class TrackLocations(Predefined): + r"""## :TrackLocations native symbol: + +
+
'$TrackLocations' +
specifies whether we should track \ + source-text location information during evaluation. This \ + can be helpful in debugging when there is a failure. +
+ """ + + name = "$TrackLocations" + messages = {"bool": "`1` should be True or False."} + + summary_text = "track source-text locations in evaluation" + + def evaluate(self, evaluation: Evaluation) -> Symbol: + print(mathics_scanner.location.MATHICS3_PATHS) + return from_bool(mathics_scanner.location.TRACK_LOCATIONS) + + def eval_set(self, value, evaluation): + """Set[$TrackLocations, value_]""" + if value is SymbolTrue or value is SymbolFalse: + evaluation.definitions.set_ownvalue("System`$TrackLocations", value) + mathics.core.parser.parser.TRACK_LOCATIONS = value.to_python() + else: + evaluation.message("$TrackLocations", "bool", value) + return value diff --git a/mathics/core/builtin.py b/mathics/core/builtin.py index c841a2a02..bab2913d8 100644 --- a/mathics/core/builtin.py +++ b/mathics/core/builtin.py @@ -484,7 +484,15 @@ def get_functions(self, prefix="eval", is_pymodule=False): definition_class = ( PyMathicsDefinitions() if is_pymodule else SystemDefinitions() ) - pattern = parse_builtin_rule(pattern, definition_class) + + # Passing the function parameter is in a way + # redundant, because creating FunctionApplyRule has + # access to the function and sets the postion this + # way. But revised afte the dust has settled and + # we have a very good idea of what is desirable and useful. + pattern = parse_builtin_rule( + pattern, definition_class, location=function + ) if unavailable_function: function = unavailable_function if attrs: diff --git a/mathics/core/expression.py b/mathics/core/expression.py index 65569ccf3..f52f1dbdd 100644 --- a/mathics/core/expression.py +++ b/mathics/core/expression.py @@ -4,6 +4,7 @@ import math from bisect import bisect_left from itertools import chain +from types import MethodType from typing import ( Any, Callable, @@ -18,6 +19,7 @@ ) import sympy +from mathics_scanner.location import SourceRange, SourceRange2 from mathics.core.atoms import Integer1, String from mathics.core.attributes import ( @@ -275,6 +277,7 @@ class Expression(BaseElement, NumericOperators, EvalMixin): elements_properties: Optional[ElementsProperties] options: Optional[Dict[str, Any]] pattern_sequence: bool + location: Optional[Union[SourceRange, SourceRange2, MethodType]] def __init__( self, @@ -300,6 +303,7 @@ def __init__( self._sequences = None self._cache = None + self.location = None # self.copy creates this self.original: Optional[Expression] = None @@ -1250,6 +1254,9 @@ def rest_range(indices): head, *elements, elements_properties=self.elements_properties ) + if hasattr(self, "location") and self.location is not None: + new.location = self.location + # Step 3: Now, process the attributes of head # If there are sequence, flatten them if the attributes allow it. if ( diff --git a/mathics/core/parser/ast.py b/mathics/core/parser/ast.py index ca9c286ab..292302f50 100644 --- a/mathics/core/parser/ast.py +++ b/mathics/core/parser/ast.py @@ -23,7 +23,7 @@ class Node: expression's leaves and a non-leaf nodes. """ - def __init__(self, head, *children): + def __init__(self, head, *children, location=None): if isinstance(head, Node): self.head = head else: @@ -31,6 +31,7 @@ def __init__(self, head, *children): self.value = None self.children = list(children) self.parenthesised = False + self.location = location def get_head_name(self): if isinstance(self.head, Symbol): @@ -68,11 +69,12 @@ class Atom(Node): their own. You can however compare Atoms for equality. """ - def __init__(self, value): + def __init__(self, value, location=None): self.head = Symbol(self.__class__.__name__) self.value = value self.children = [] self.parenthesised = False + self.location = location def __repr__(self): return "%s[%s]" % (self.head, self.value) @@ -90,7 +92,13 @@ class Number(Atom): """ def __init__( - self, value: str, sign: int = 1, base: int = 10, suffix=None, exp: int = 0 + self, + value: str, + sign: int = 1, + base: int = 10, + suffix=None, + exp: int = 0, + location=None, ): assert isinstance(value, str) assert sign in (-1, 1) @@ -104,6 +112,7 @@ def __init__( self.base = base self.suffix = suffix self.exp = exp + self.location = location def __repr__(self): result = self.value @@ -132,10 +141,11 @@ class Symbol(Atom): are unique as they are say in Lisp, or Python. """ - def __init__(self, value: str, context: Optional[str] = "System"): + def __init__(self, value: str, context: Optional[str] = "System", location=None): self.context = context self.value = value self.children = [] + self.location = location # avoids recursive definition @property diff --git a/mathics/core/parser/feed.py b/mathics/core/parser/feed.py index fba41e37f..66160b165 100644 --- a/mathics/core/parser/feed.py +++ b/mathics/core/parser/feed.py @@ -21,7 +21,7 @@ def send_messages(self, evaluation) -> list: class MathicsSingleLineFeeder(SingleLineFeeder, MathicsLineFeeder): - "A feeder that feeds lines from an open ``File`` object" + "A feeder that feeds lines from an open location container object" class MathicsFileLineFeeder(FileLineFeeder, MathicsLineFeeder): @@ -29,4 +29,4 @@ class MathicsFileLineFeeder(FileLineFeeder, MathicsLineFeeder): class MathicsMultiLineFeeder(MultiLineFeeder, MathicsLineFeeder): - "A feeder that feeds lines from an open ``File`` object" + "A feeder that feeds lines from an open contianer object" diff --git a/mathics/core/parser/location.py b/mathics/core/parser/location.py new file mode 100644 index 000000000..a4c9275ff --- /dev/null +++ b/mathics/core/parser/location.py @@ -0,0 +1,116 @@ +""" +Provides location tracking in parser. +""" + +from typing import Callable, Optional + +import mathics_scanner +from mathics_scanner.location import ( + EVAL_METHODS, + ContainerKind, + SourceRange, + SourceRange2, +) + +from mathics.core.parser.ast import Node + + +def track_location(func: Callable) -> Callable: + """Python decorator for a parse method that adds location tracking + on non-leaf-nodes when TRACK_LOCATION is set. Otherwise, we just + run the parse method. + + """ + + def wrapper(self, *args) -> Optional[Node]: + if not mathics_scanner.location.TRACK_LOCATIONS: + return func(self, *args) + + # Save location information + # For expressions which make their way to + # FunctionApplyRule, saving a position here is + # extraneous because the FunctionApplyRule is + # the position. But deal with this redundancy + # after the dust settles, and we have experience + # on what is desired. + start_column = self.tokeniser.pos + start_line = self.feeder.lineno + parsed_node = func(self, *args) + + if parsed_node is not None and hasattr(parsed_node, "location"): + if self.feeder.container_kind == ContainerKind.PYTHON: + parsed_node.location = self.feeder.container + if self.feeder.container not in EVAL_METHODS: + EVAL_METHODS.add(parsed_node.location) + else: + end_pos = self.tokeniser.pos + end_line = self.feeder.lineno + if start_line == end_line: + parsed_node.location = SourceRange2( + start_line=start_line, + start_pos=start_column, + end_pos=end_pos, + container=self.feeder.container_index, + ) + else: + parsed_node.location = SourceRange( + start_line=start_line, + start_pos=start_column, + end_line=end_line, + end_pos=end_pos, + container=self.feeder.container_index, + ) + + return parsed_node + + return wrapper + + +def track_token_location(func: Callable) -> Callable: + """Python decorator for a parse method that adds location + tracking to leaf-nodes, i.e. tokens. This happens though only TRACK_LOCATION is set. Otherwise, we just run the + normal parse method. + + """ + + def wrapper(self, token) -> Optional[Node]: + if not mathics_scanner.location.TRACK_LOCATIONS: + return func(self, token) + + # Save location information + # For expressions which make their way to + # FunctionApplyRule, saving a position here is + # extraneous because the FunctionApplyRule is + # the position. But deal with this redundancy + # after the dust settles, and we have experience + # on what is desired. + start_column = token.pos + start_line = self.feeder.lineno + parsed_node = func(self, token) + + if ( + self.feeder.container_kind == ContainerKind.PYTHON + and self.feeder.container not in EVAL_METHODS + ): + location = self.feeder.container + EVAL_METHODS.add(location) + + end_line = self.feeder.lineno + if start_line == end_line: + parsed_node.location = SourceRange2( + start_line=start_line, + start_pos=start_column, + end_pos=start_column + len(token.text) - 1, + container=self.feeder.container_index, + ) + else: + parsed_node.location = SourceRange( + start_line=start_line, + start_pos=start_column, + end_line=self.feeder.lineno, + end_pos=start_column + len(token.text) - 1, + container=self.feeder.container_index, + ) + return parsed_node + + return wrapper diff --git a/mathics/core/parser/parser.py b/mathics/core/parser/parser.py index f7e2afa5f..a53679f8f 100644 --- a/mathics/core/parser/parser.py +++ b/mathics/core/parser/parser.py @@ -30,6 +30,7 @@ String, Symbol, ) +from mathics.core.parser.location import track_location, track_token_location from mathics.core.parser.operators import ( all_operators, binary_operators, @@ -97,6 +98,7 @@ def __init__(self): "DifferentialD", ] ) + self.location = None def backtrack(self, pos): """ @@ -166,8 +168,10 @@ def parse(self, feeder) -> Optional[Node]: self.current_token = None self.bracket_depth = 0 self.box_depth = 0 + return self.parse_e() + @track_location def parse_e(self) -> Optional[Node]: """ Parse the single top-level or "start" expression. @@ -183,6 +187,7 @@ def parse_e(self) -> Optional[Node]: else: return None + @track_location def parse_binary_operator( self, expr1, token: Token, expr1_precedence: int ) -> Optional[Node]: @@ -354,6 +359,7 @@ def parse_box_operator( return result + @track_location def parse_comparison( self, expr1, token: Token, expr1_precedence: int ) -> Optional[Node]: @@ -409,6 +415,7 @@ def parse_comparison( expr1 = Node(tag, expr1, expr2).flatten() return expr1 + @track_location def parse_expr(self, precedence: int) -> Optional[Node]: """ Parse an expression returning an AST Node tree for this. @@ -502,6 +509,7 @@ def parse_expr(self, precedence: int) -> Optional[Node]: result = new_result return result + @track_location def parse_p(self): """Parse a "p_"-tagged expression. "p_" tags include prefix operators, left-bracketed expressions @@ -552,6 +560,7 @@ def parse_postfix( # Note: returning a list is different from how most other parse_ routines # work and it makes the type system more complicated. + @track_location def parse_seq(self) -> list: result: list = [] while True: @@ -713,7 +722,7 @@ def e_Alternatives(self, expr1, token: Token, p: int) -> Optional[Node]: expr2 = self.parse_expr(q + 1) return Node("Alternatives", expr1, expr2).flatten() - def e_ApplyList(self, expr1, token: Token, p: int) -> Optional[Node]: + def e_ApplyList(self, expr1, _: Token, p: int) -> Optional[Node]: operator_precedence = right_binary_operators["Apply"] if operator_precedence < p: return None @@ -722,7 +731,7 @@ def e_ApplyList(self, expr1, token: Token, p: int) -> Optional[Node]: expr3 = Node("List", Number1) return Node("Apply", expr1, expr2, expr3) - def e_Derivative(self, expr1, token: Token, p: int) -> Optional[Node]: + def e_Derivative(self, expr1, _: Token, p: int) -> Optional[Node]: q = postfix_operators["Derivative"] if q < p: return None @@ -733,7 +742,7 @@ def e_Derivative(self, expr1, token: Token, p: int) -> Optional[Node]: head = Node("Derivative", Number(str(n))) return Node(head, expr1) - def e_Divide(self, expr1, token: Token, expr1_precedence: int): + def e_Divide(self, expr1, _: Token, expr1_precedence: int): """ Implements parsing and transformation of Divide expr1 / expr2 @@ -768,7 +777,8 @@ def e_Divide(self, expr1, token: Token, expr1_precedence: int): expr2 = self.parse_expr(operator_precedence + 1) return Node("Times", expr1, Node("Power", expr2, NumberM1)).flatten() - def e_Infix(self, expr1, token: Token, expr1_precedence) -> Optional[Node]: + @track_location + def e_Infix(self, expr1, _: Token, expr1_precedence) -> Optional[Node]: """ Used to implement the rule: expr : expr1 '~' expr2 '~' expr3 @@ -861,7 +871,7 @@ def e_Prefix(self, expr1, _: Token, expr1_precedence: int) -> Optional[Node]: expr2 = self.parse_expr(operator_precedence) return Node(expr1, expr2) - def e_Postfix(self, expr1, token: Token, expr1_precedence: int) -> Optional[Node]: + def e_Postfix(self, expr1, _: Token, expr1_precedence: int) -> Optional[Node]: """ Used to parse expr1 // expr2 @@ -891,7 +901,7 @@ def e_Postfix(self, expr1, token: Token, expr1_precedence: int) -> Optional[Node expr2 = self.parse_expr(operator_precedence + 1) return Node(expr2, expr1) - def e_RawColon(self, expr1, token: Token, p: int) -> Optional[Node]: + def e_RawColon(self, expr1, _: Token, p: int) -> Optional[Node]: head_name = expr1.get_head_name() if head_name == "Symbol": head = "Pattern" @@ -912,6 +922,7 @@ def e_RawColon(self, expr1, token: Token, p: int) -> Optional[Node]: expr2 = self.parse_expr(q + 1) return Node(head, expr1, expr2) + @track_location def e_RawLeftBracket(self, expr, token: Token, p: int) -> Optional[Node]: q = all_operators["Part"] if q < p: @@ -940,7 +951,8 @@ def e_RawLeftBracket(self, expr, token: Token, p: int) -> Optional[Node]: result.parenthesised = True return result - def e_Semicolon(self, expr1, token: Token, expr1_precedence: int) -> Optional[Node]: + @track_location + def e_Semicolon(self, expr1, _: Token, expr1_precedence: int) -> Optional[Node]: """ Used to parse expr1 ; expr2 @@ -985,6 +997,7 @@ def e_Semicolon(self, expr1, token: Token, expr1_precedence: int) -> Optional[No expr2 = NullSymbol return Node("CompoundExpression", expr1, expr2).flatten() + @track_location def e_Span(self, expr1, token: Token, p) -> Optional[Node]: q = ternary_operators["Span"] if q < p: @@ -1046,7 +1059,8 @@ def e_TagSet(self, expr1, token: Token, p: int) -> Optional[Node]: expr3 = self.parse_expr(q + 1) return Node(head, expr1, expr2, expr3) - def e_Unset(self, expr1, token: Token, p: int) -> Optional[Node]: + @track_location + def e_Unset(self, expr1, _: Token, p: int) -> Optional[Node]: q = all_operators["Set"] if q < p: return None @@ -1061,17 +1075,17 @@ def e_Unset(self, expr1, token: Token, p: int) -> Optional[Node]: # can uniquely identified by a prefix character or string. # FIXME DRY with pre_Decrement - def p_Decrement(self, token: Token) -> Node: + def p_Decrement(self, _: Token) -> Node: self.consume() q = prefix_operators["PreDecrement"] return Node("PreDecrement", self.parse_expr(q)) - def p_Increment(self, token: Token) -> Node: + def p_Increment(self, _: Token) -> Node: self.consume() q = prefix_operators["PreIncrement"] return Node("PreIncrement", self.parse_expr(q)) - def p_Information(self, token: Token) -> Node: + def p_Information(self, _: Token) -> Node: self.consume() q = prefix_operators["Information"] child = self.parse_expr(q) @@ -1081,7 +1095,7 @@ def p_Information(self, token: Token) -> Node: "Information", child, Node("Rule", Symbol("LongForm"), Symbol("True")) ) - def p_Integral(self, token: Token) -> Node: + def p_Integral(self, _: Token) -> Node: self.consume() inner_prec, outer_prec = all_operators["Sum"] + 1, all_operators["Power"] - 1 expr1 = self.parse_expr(inner_prec) @@ -1089,17 +1103,19 @@ def p_Integral(self, token: Token) -> Node: expr2 = self.parse_expr(outer_prec) return Node("Integrate", expr1, expr2) - def p_Factorial2(self, token: Token) -> Node: + def p_Factorial2(self, _: Token) -> Node: self.consume() q = prefix_operators["Not"] child = self.parse_expr(q) return Node("Not", Node("Not", child)) + @track_token_location def p_Filename(self, token: Token) -> Filename: result = Filename(token.text) self.consume() return result + @track_token_location def p_LeftRowBox(self, token: Token) -> Union[Node, String]: self.consume() children = [] @@ -1123,7 +1139,8 @@ def p_LeftRowBox(self, token: Token) -> Union[Node, String]: result.parenthesised = True return result - def p_Minus(self, token: Token) -> Optional[Node]: + @track_token_location + def p_Minus(self, _: Token) -> Optional[Node]: """ Used to parse: - expr1 @@ -1140,7 +1157,8 @@ def p_Minus(self, token: Token) -> Optional[Node]: else: return Node("Times", NumberM1, expr).flatten() - def p_MinusPlus(self, token: Token) -> Node: + @track_token_location + def p_MinusPlus(self, _: Token) -> Node: """ Used to parse: ∓ expr1 @@ -1153,7 +1171,8 @@ def p_MinusPlus(self, token: Token) -> Node: operator_precedence = operator_precedences["UnaryMinusPlus"] return Node("MinusPlus", self.parse_expr(operator_precedence)) - def p_Not(self, token: Token) -> Node: + @track_token_location + def p_Not(self, _: Token) -> Node: self.consume() operator_precedence = prefix_operators["Not"] child = self.parse_expr(operator_precedence) @@ -1165,6 +1184,7 @@ def p_Not(self, token: Token) -> Node: # See if we can fix this mess. p_Factorial = p_Not + @track_token_location def p_Number(self, token: Token) -> Number: s = token.text @@ -1260,7 +1280,8 @@ def p_PatternTest(self, token: Token) -> Node: "Information", child, Node("Rule", Symbol("LongForm"), Symbol("False")) ) - def p_Plus(self, token: Token): + @track_token_location + def p_Plus(self, _: Token): """ Used to parse: + expr1 diff --git a/mathics/core/parser/util.py b/mathics/core/parser/util.py index 22d47f1fc..ece35aa21 100644 --- a/mathics/core/parser/util.py +++ b/mathics/core/parser/util.py @@ -2,7 +2,9 @@ from typing import FrozenSet, Optional, Tuple +import mathics_scanner.location from mathics_scanner.feed import LineFeeder +from mathics_scanner.location import ContainerKind from mathics.core.definitions import Definitions from mathics.core.element import BaseElement @@ -64,17 +66,22 @@ def parse_returning_code( methods. See the mathics_scanner.feed module. """ - from mathics.core.expression import Expression - ast = parser.parse(feeder) source_text = parser.tokeniser.source_text + if ( + mathics_scanner.location.TRACK_LOCATIONS + and feeder.container_kind == ContainerKind.STREAM + ): + feeder.container.append(source_text) if ast is None: return None, source_text converted = convert(ast, definitions) + if hasattr(converted, "location") and ast.location: + converted.location = ast.location return converted, source_text @@ -124,9 +131,12 @@ def lookup_name(self, name): return ensure_context(name, context) -def parse_builtin_rule(string, definitions=SystemDefinitions()): +def parse_builtin_rule(string, definitions=SystemDefinitions(), location=None): """ Parse rules specified in builtin docstrings/attributes. Every symbol in the input is created in the System` context. """ - return parse(definitions, MathicsSingleLineFeeder(string, "")) + return parse( + definitions, + MathicsSingleLineFeeder(string, location, ContainerKind.PYTHON), + ) diff --git a/mathics/core/pattern.py b/mathics/core/pattern.py index 9047b2ec1..ac5a67bf9 100644 --- a/mathics/core/pattern.py +++ b/mathics/core/pattern.py @@ -427,6 +427,7 @@ def __init__( evaluation: Optional[Evaluation] = None, ): self.expr = expr + self.location = expr.location if hasattr(expr, "location") else None head = expr.head if attributes is None and evaluation: attributes = head.get_attributes(evaluation.definitions) diff --git a/mathics/core/rules.py b/mathics/core/rules.py index 1af7ffddc..18e67844c 100644 --- a/mathics/core/rules.py +++ b/mathics/core/rules.py @@ -98,6 +98,7 @@ def __init__( evaluation: Optional[Evaluation] = None, attributes: Optional[int] = None, ) -> None: + self.location: Optional[Callable] = None self.pattern = BasePattern.create( pattern, attributes=attributes, evaluation=evaluation ) @@ -179,6 +180,12 @@ def yield_match(vars, rest): expr._elements_fully_evaluated = False expr._is_flat = False # I think this is fully updated expr._is_ordered = False + if ( + hasattr(expression, "location") + and hasattr(expr, "location") + and expression.location is not None + ): + expr.location = expression.location return expr if return_list: @@ -280,44 +287,44 @@ def __repr__(self) -> str: class FunctionApplyRule(BaseRule): """ - A FunctionApplyRule is a rule that has a replacement term that - is associated a Python function rather than a Mathics Expression - as happens in a transformation Rule. + A FunctionApplyRule is a rule that has a replacement term that + is associated a Python function rather than a Mathics Expression + as happens in a transformation Rule. - Each time the Pattern part of the Rule matches an Expression, the - matching subexpression is replaced by the expression returned - by application of that function to the remaining terms. + Each time the Pattern part of the Rule matches an Expression, the + matching subexpression is replaced by the expression returned + by application of that function to the remaining terms. - Parameters for the function are bound to parameters matched by the pattern. + Parameters for the function are bound to parameters matched by the pattern. - Here is an example taken from the symbol ``System`Plus``. - It has has associated a FunctionApplyRule:: + Here is an example taken from the symbol ``System`Plus``. + It has has associated a FunctionApplyRule:: - Plus[items___] -> mathics.builtin.arithfns.basic.Plus.apply + Plus[items___] -> mathics.builtin.arithfns.basic.Plus.apply - The pattern ``items___`` matches a list of Expressions. + The pattern ``items___`` matches a list of Expressions. - When applied to the expression ``F[a+a]`` the method - ``mathics.builtin.arithfns.basic.Plus.apply`` is called - binding the parameter ``items`` to the value ``Sequence[a,a]``. + When applied to the expression ``F[a+a]`` the method + ``mathics.builtin.arithfns.basic.Plus.apply`` is called + binding the parameter ``items`` to the value ``Sequence[a,a]``. - The return value of this function is ``Times[2, a]`` (or more compactly: ``2*a``). - When replaced in the original expression, the result is: ``F[2*a]``. + The return value of this function is ``Times[2, a]`` (or more compactly: ``2*a``). + When replaced in the original expression, the result is: ``F[2*a]``. - In contrast to (transformation) Rules, FunctionApplyRules can - change the state of definitions in the the system. + In contrast to (transformation) Rules, FunctionApplyRules can + change the state of definitions in the the system. + p + For example, the rule:: - For example, the rule:: + SetAttributes[a_,b_] -> mathics.builtin.attributes.SetAttributes.apply - SetAttributes[a_,b_] -> mathics.builtin.attributes.SetAttributes.apply + when applied to the expression ``SetAttributes[F, NumericFunction]`` - when applied to the expression ``SetAttributes[F, NumericFunction]`` + sets the attribute ``NumericFunction`` in the definition of the symbol + ``F`` and returns Null (``SymbolNull``). - sets the attribute ``NumericFunction`` in the definition of the symbol - ``F`` and returns Null (``SymbolNull``). - - This will cause `Expression.evaluate() to perform an additional - ``rewrite_apply_eval()`` step. + This will cause `Expression.evaluate() to perform an additional + ``rewrite_apply_eval()`` step. """ @@ -335,7 +342,7 @@ def __init__( pattern, system=system, attributes=attributes, evaluation=evaluation ) self.name = name - self.function = function + self.location = self.function = function self.check_options = check_options # If you update this, you must also update traced_apply_function diff --git a/mathics/eval/files_io/files.py b/mathics/eval/files_io/files.py index 93b196f57..46f2706b5 100644 --- a/mathics/eval/files_io/files.py +++ b/mathics/eval/files_io/files.py @@ -11,6 +11,7 @@ InvalidSyntaxError, SyntaxError, ) +from mathics_scanner.location import ContainerKind import mathics import mathics.core.parser @@ -271,7 +272,9 @@ def eval_Read( assert isinstance(tmp, str) while True: try: - feeder = MathicsMultiLineFeeder(tmp) + feeder = MathicsMultiLineFeeder( + tmp, "", ContainerKind.STREAM + ) expr = parse_incrementally_by_line( evaluation.definitions, feeder ) diff --git a/mathics/eval/tracing.py b/mathics/eval/tracing.py index 06d85b211..d458f41f4 100644 --- a/mathics/eval/tracing.py +++ b/mathics/eval/tracing.py @@ -17,6 +17,19 @@ hook_exit_fn: Optional[Callable] = None +def is_performing_rewrite(func) -> bool: + """ " + Returns true if we are in the rewrite expression phase + as opposed to the apply-function/evaluation phase of + evaluation. The way we determine this is highly specific + to the Mathics3 code as it stands right now. So this + code is highly fragile and can change when the + evaluation code changes. However encapsulating this + in a function helps narrows the fragility to one place. + """ + return hasattr(func, "__name__") and func.__name__ == "rewrite_apply_eval_step" + + def skip_trivial_evaluation(expr, status: str, orig_expr=None) -> bool: """ Look for uninteresting evaluations that we should avoid showing @@ -85,8 +98,7 @@ def print_evaluate(expr, evaluation, status: str, fn: Callable, orig_expr=None): indents = " " * evaluation.recursion_depth if orig_expr is not None: - fn_name = fn.__name__ if hasattr(fn, "__name__") else None - if fn_name == "rewrite_apply_eval_step": + if is_performing_rewrite(fn): assert isinstance(expr, tuple) if orig_expr != expr[0]: if status == "Returning": @@ -103,9 +115,12 @@ def print_evaluate(expr, evaluation, status: str, fn: Callable, orig_expr=None): f"{indents}{status}: {expr[0]}" + arrow + str(expr) ) else: + if status == "Returning" and isinstance(expr, tuple): + status = "Evaluating/Replacing" + expr = expr[0] evaluation.print_out(f"{indents}{status}: {orig_expr} = " + str(expr)) - elif fn.__name__ != "rewrite_apply_eval_step": + elif not is_performing_rewrite(fn): evaluation.print_out(f"{indents}{status}: {expr}") return @@ -140,9 +155,10 @@ def wrapper(expr, evaluation) -> Any: ): # We may use boxing in print_evaluate_fn(). So turn off # boxing temporarily. + phase_name = "Rewriting" if is_performing_rewrite(func) else "Evaluating" evaluation.is_boxing = True trace_evaluate_action = trace_evaluate_on_call( - expr, evaluation, "Evaluating", func + expr, evaluation, phase_name, func ) evaluation.is_boxing = was_boxing if trace_evaluate_action is None: @@ -158,14 +174,14 @@ def wrapper(expr, evaluation) -> Any: if trace_evaluate_action is not None: result = ( (trace_evaluate_action, False) - if func.__name__ == "rewrite_apply_eval_step" + if is_performing_rewrite(func) else trace_evaluate_action ) evaluation.is_boxing = was_boxing else: result = ( (trace_evaluate_action, False) - if func.__name__ == "rewrite_apply_eval_step" + if is_performing_rewrite(func) else trace_evaluate_action ) return result diff --git a/mathics/main.py b/mathics/main.py index 6b8dc538c..d910e4918 100755 --- a/mathics/main.py +++ b/mathics/main.py @@ -19,6 +19,9 @@ import sys from typing import List +import mathics_scanner.location +from mathics_scanner.location import ContainerKind + import mathics.core as mathics_core from mathics import __version__, license_string, settings, version_string from mathics.builtin.trace import TraceBuiltins, traced_apply_function @@ -79,7 +82,7 @@ def __init__( in_prefix: str = "In", out_prefix: str = "Out", ): - super(TerminalShell, self).__init__("") + super(TerminalShell, self).__init__([], ContainerKind.STREAM) self.input_encoding = locale.getpreferredencoding() self.lineno = 0 self.in_prefix = in_prefix @@ -250,6 +253,8 @@ def reset_lineno(self): def feed(self): result = self.read_line(self.get_in_prompt()) + "\n" + if mathics_scanner.location.TRACK_LOCATIONS: + self.container.append(self.source_text) if result == "\n": return "" # end of input self.lineno += 1 diff --git a/mathics/session.py b/mathics/session.py index cb566a5a0..807e5487e 100644 --- a/mathics/session.py +++ b/mathics/session.py @@ -14,6 +14,8 @@ from os.path import join as osp_join from typing import Optional +from mathics_scanner.location import ContainerKind + from mathics.core.definitions import Definitions from mathics.core.evaluation import Evaluation, Result from mathics.core.parser import MathicsSingleLineFeeder, parse @@ -140,7 +142,10 @@ def reset(self, add_builtin=True, catch_interrupt=False): def evaluate(self, str_expression, timeout=None, form=None): """Parse str_expression and evaluate using the `evaluate` method of the Expression""" self.evaluation.out.clear() - expr = parse(self.definitions, MathicsSingleLineFeeder(str_expression)) + expr = parse( + self.definitions, + MathicsSingleLineFeeder(str_expression, ContainerKind.STREAM), + ) if form is None: form = self.form self.last_result = expr.evaluate(self.evaluation) diff --git a/test/builtin/test_trace.py b/test/builtin/test_trace.py index 8a74e0b61..a5c159371 100644 --- a/test/builtin/test_trace.py +++ b/test/builtin/test_trace.py @@ -33,7 +33,7 @@ def counting_print_evaluate( """ global trace_evaluation_calls trace_evaluation_calls += 1 - assert status in ("Evaluating", "Returning") + assert status in ("Evaluating", "Returning", "Rewriting") if "cython" not in version_info: assert isfunction(fn), "Expecting 4th argument to be a function" return None @@ -119,9 +119,9 @@ def capture_print(s: str): assert [ " Evaluating: System`Plus[System`Times[2, 3], 4]", " Evaluating: System`Times[2, 3]", - " Returning: System`Times[2, 3] = (, False)", + " Evaluating/Replacing: System`Times[2, 3] = 6", " Returning: System`Times[2, 3] = 6", - " Returning: System`Plus[System`Times[2, 3], 4] = (, False)", + " Evaluating/Replacing: System`Plus[System`Times[2, 3], 4] = 10", " Returning: System`Plus[System`Times[2, 3], 4] = 10", ] == event_queue # print() diff --git a/test/core/parser/test_box_parser.py b/test/core/parser/test_box_parser.py index 7a64dbeab..c6c6dc28a 100644 --- a/test/core/parser/test_box_parser.py +++ b/test/core/parser/test_box_parser.py @@ -7,6 +7,7 @@ from typing import Optional from mathics_scanner import SingleLineFeeder +from mathics_scanner.location import ContainerKind from mathics.core.parser.parser import Parser @@ -14,12 +15,14 @@ # Note we don't import mathics.session here since we # are testing just the parse layer, not the evaluation layer. # Simpler is better. -parser = Parser() +core_parser = Parser() def check_evaluation(str_expr: str, str_expected: str, assert_message: Optional[str]): - def parse(s: str): - return parser.parse(SingleLineFeeder(s)) + def parse(source_text: str): + return core_parser.parse( + SingleLineFeeder(source_text, "", ContainerKind.STRING) + ) result = parse(str_expr) expected = parse(str_expected) diff --git a/test/core/parser/test_convert.py b/test/core/parser/test_convert.py index f1f1bd73f..a7010764d 100644 --- a/test/core/parser/test_convert.py +++ b/test/core/parser/test_convert.py @@ -8,20 +8,26 @@ InvalidSyntaxError, SyntaxError, ) +from mathics_scanner.location import ContainerKind from mathics.core.atoms import Integer, Integer0, Integer1, Rational, Real, String from mathics.core.definitions import Definitions from mathics.core.expression import Expression -from mathics.core.parser import parse +from mathics.core.load_builtin import import_and_load_builtins +from mathics.core.parser import parse as core_parse from mathics.core.symbols import Symbol from mathics.core.systemsymbols import SymbolDerivative +import_and_load_builtins() definitions = Definitions(add_builtin=True) class ConvertTests(unittest.TestCase): - def parse(self, code): - return parse(definitions, SingleLineFeeder(code)) + def parse(self, source_text): + return core_parse( + definitions, + SingleLineFeeder(source_text, "", ContainerKind.STRING), + ) def check(self, expr1, expr2): if isinstance(expr1, str): diff --git a/test/core/parser/test_parser.py b/test/core/parser/test_parser.py index 274165fe4..378306c31 100644 --- a/test/core/parser/test_parser.py +++ b/test/core/parser/test_parser.py @@ -12,6 +12,7 @@ NamedCharacterSyntaxError, SyntaxError, ) +from mathics_scanner.location import ContainerKind from mathics.core.parser.ast import Filename, Node, Number, String, Symbol from mathics.core.parser.parser import Parser @@ -22,7 +23,9 @@ def setUp(self): self.parser = Parser() def parse(self, s: str): - return self.parser.parse(SingleLineFeeder(s)) + return self.parser.parse( + SingleLineFeeder(s, "", ContainerKind.STRING) + ) def check(self, expr1, expr2): if isinstance(expr1, str): diff --git a/test/core/parser/test_util.py b/test/core/parser/test_util.py index dc6ec330b..e4e1652f5 100644 --- a/test/core/parser/test_util.py +++ b/test/core/parser/test_util.py @@ -7,15 +7,18 @@ SingleLineFeeder, SyntaxError, ) +from mathics_scanner.location import ContainerKind from mathics.core.definitions import Definitions -from mathics.core.parser import parse +from mathics.core.load_builtin import import_and_load_builtins +from mathics.core.parser import parse as core_parse +import_and_load_builtins() definitions = Definitions(add_builtin=True) class UtilTests(unittest.TestCase): - def parse(self, code): + def parse(self, source_text: str): raise NotImplementedError def compare(self, expr1, expr2): @@ -43,8 +46,11 @@ def syntax_error(self, string): class SingleLineParserTests(UtilTests): - def parse(self, code): - return parse(definitions, SingleLineFeeder(code)) + def parse(self, source_text): + return core_parse( + definitions, + SingleLineFeeder(source_text, "", ContainerKind.STRING), + ) def compare(self, expr1, expr2): assert expr1.sameQ(expr2) @@ -63,8 +69,13 @@ def test_trailing_backslash(self): class MultiLineParserTests(UtilTests): - def parse(self, code): - return parse(definitions, MultiLineFeeder(code)) + def parse(self, source_text): + return core_parse( + definitions, + MultiLineFeeder( + source_text, "", ContainerKind.STRING + ), + ) def compare(self, expr1, expr2): assert expr1.sameQ(expr2) @@ -89,14 +100,20 @@ def test_CompoundExpression(self): self.check("a;^b", "Power[CompoundExpression[a, Null], b]") - feeder = MultiLineFeeder("a;\n^b") + feeder = MultiLineFeeder( + "a;\n^b", "", ContainerKind.STRING + ) self.compare( - parse(definitions, feeder), self.parse("CompoundExpression[a, Null]") + core_parse(definitions, feeder), self.parse("CompoundExpression[a, Null]") + ) + self.assertRaises( + InvalidSyntaxError, lambda f: core_parse(definitions, f), feeder ) - self.assertRaises(InvalidSyntaxError, lambda f: parse(definitions, f), feeder) def test_Span(self): self.check("a;;^b", "Power[Span[a, All], b]") - feeder = MultiLineFeeder("a;;\n^b") - self.compare(parse(definitions, feeder), self.parse("Span[a, All]")) - self.assertRaises(InvalidSyntaxError, lambda f: parse(definitions, f), feeder) + feeder = MultiLineFeeder("a;;\n^b", "", ContainerKind.STRING) + self.compare(core_parse(definitions, feeder), self.parse("Span[a, All]")) + self.assertRaises( + InvalidSyntaxError, lambda f: core_parse(definitions, f), feeder + ) diff --git a/test/core/test_elements_properties.py b/test/core/test_elements_properties.py index cdacb77a6..6254ffba0 100644 --- a/test/core/test_elements_properties.py +++ b/test/core/test_elements_properties.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- +from mathics_scanner.location import ContainerKind + from mathics.core.parser import MathicsSingleLineFeeder from mathics.core.parser.convert import convert from mathics.core.parser.parser import Parser @@ -40,7 +42,9 @@ def test_elements_properties(): ]: # fmt: on session.evaluation.out.clear() - feeder = MathicsSingleLineFeeder(str_expression) + feeder = MathicsSingleLineFeeder( + str_expression, "", ContainerKind.STRING + ) ast = parser.parse(feeder) # convert() creates the initial Expression. In that various properties should diff --git a/test/core/test_is_literal.py b/test/core/test_is_literal.py index 439e8fb58..63b932696 100644 --- a/test/core/test_is_literal.py +++ b/test/core/test_is_literal.py @@ -3,6 +3,8 @@ Test mathics.core property on BaseElement is_literal. """ +from mathics_scanner.location import ContainerKind + from mathics.core.parser import MathicsSingleLineFeeder from mathics.core.parser.convert import convert from mathics.core.parser.parser import Parser @@ -33,7 +35,9 @@ def test_is_literal(): ]: # fmt: on session.evaluation.out.clear() - feeder = MathicsSingleLineFeeder(str_expression) + feeder = MathicsSingleLineFeeder( + str_expression, "", ContainerKind.STRING + ) ast = parser.parse(feeder) # print("XXX", ast) diff --git a/test/eval/test_patterns.py b/test/eval/test_patterns.py index 865080f06..1bb1e2bd4 100644 --- a/test/eval/test_patterns.py +++ b/test/eval/test_patterns.py @@ -6,6 +6,7 @@ from test.helper import session import pytest +from mathics_scanner.location import ContainerKind from mathics.core.definitions import Definitions from mathics.core.parser import MathicsSingleLineFeeder, parse @@ -17,8 +18,18 @@ def check_pattern(str_expr, str_pattern): - expr = parse(defintions, MathicsSingleLineFeeder(str_expr)) - pattern = ExpressionPattern(parse(defintions, MathicsSingleLineFeeder(str_pattern))) + expr = parse( + defintions, + MathicsSingleLineFeeder(str_expr, "", ContainerKind.STRING), + ) + pattern = ExpressionPattern( + parse( + defintions, + MathicsSingleLineFeeder( + str_pattern, "", ContainerKind.STRING + ), + ) + ) ret = Matcher(pattern, session.evaluation).match(expr, session.evaluation) assert ret is True