Revise API for position tracking

.github/workflows/consistency-checks.yml

@@ -23,7 +23,7 @@ jobs:
         sudo apt update -qq && sudo apt install llvm-dev remake
         python -m pip install --upgrade pip
         # We can comment out after next Mathics-Scanner release
-        python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full]
+        python -m pip install -e git+https://github.com/Mathics3/mathics-scanner@position-tracking#egg=Mathics-Scanner[full]
         pip install -e .
 
     - name: Install Mathics with minimum dependencies

.github/workflows/macos.yml

@@ -35,7 +35,7 @@ jobs:
         cd ..
         # We can comment out after next Mathics-Scanner release
         # python -m pip install Mathics-Scanner[full]
-        python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full]
+        python -m pip install -e git+https://github.com/Mathics3/mathics-scanner@position-tracking#egg=Mathics-Scanner[full]
         pip install -e .
         remake -x develop-full
     - name: Test Mathics3

.github/workflows/mypy.yml

@@ -27,7 +27,7 @@ jobs:
       run: |
         pip install mypy==1.13 sympy==1.12
         # Adjust below for right branch
-        git clone --depth 1 https://github.com/Mathics3/mathics-scanner.git
+        git clone --depth 1 --branch position-tracking https://github.com/Mathics3/mathics-scanner
         cd mathics-scanner/
         pip install -e .
         bash ./admin-tools/make-JSON-tables.sh

.github/workflows/packages.yml

@@ -25,7 +25,7 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         # We can comment out after next Mathics-Scanner release
-        python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full]
+        python -m pip install -e git+https://github.com/Mathics3/mathics-scanner@position-tracking#egg=Mathics-Scanner[full]
     - name: Run Mathics3 Combinatorica tests
       run: |
         git submodule init

.github/workflows/pyodide.yml

@@ -55,7 +55,7 @@ jobs:
           pip install "setuptools>=70.0.0" PyYAML click packaging pytest
 
           # We can comment out after next Mathics-Scanner release
-          python -m pip install --no-build-isolation -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner
+          python -m pip install --no-build-isolation -e git+https://github.com/Mathics3/mathics-scanner@position-tracking#egg=Mathics-Scanner
           # pip install --no-build-isolation -e .
           # cd ..
 

.github/workflows/ubuntu-cython.yml

@@ -30,7 +30,7 @@ jobs:
         pip install -e .
         cd ..
         # We can comment out after next Mathics-Scanner release
-        python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full]
+        python -m pip install -e git+https://github.com/Mathics3/mathics-scanner@position-tracking#egg=Mathics-Scanner[full]
         pip install -e .
         cd ..
 

.github/workflows/ubuntu.yml

@@ -31,7 +31,7 @@ jobs:
         cd ..
         # We can comment out after next Mathics-Scanner release
         # python -m pip install Mathics-Scanner[full]
-        python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full]
+        python -m pip install -e git+https://github.com/Mathics3/mathics-scanner@position-tracking#egg=Mathics-Scanner[full]
         pip install -e .
         remake -x develop-full
     - name: Test Mathics

.github/workflows/windows.yml

@@ -39,7 +39,7 @@ jobs:
         pip install -e .
         cd ..
         # We can comment out after next Mathics-Scanner release
-        python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full]
+        python -m pip install -e git+https://github.com/Mathics3/mathics-scanner@position-tracking#egg=Mathics-Scanner[full]
         pip install -e .
 
         # python -m pip install Mathics-Scanner[full]

examples/symbolic_logic/gries_schneider/test_gs.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
+from mathics_scanner.location import ContainerKind
 
 from mathics.core.definitions import Definitions
 from mathics.core.evaluation import Evaluation
@@ -13,5 +14,8 @@
 for i in range(0, 4):
     evaluation = Evaluation(definitions=definitions, catch_interrupt=False)
 
-    expr = parse(definitions, MathicsSingleLineFeeder(f"<< GS{i}.m"))
+    expr = parse(
+        definitions,
+        MathicsSingleLineFeeder(f"<< GS{i}.m", "<test_gs>", ContainerKind.STRING),
+    )
     expr.evaluate(evaluation)

mathics/builtin/testing_expressions/string_tests.py

@@ -5,6 +5,7 @@
 import re
 
 from mathics_scanner import SingleLineFeeder, SyntaxError
+from mathics_scanner.location import ContainerKind
 
 from mathics.builtin.atomic.strings import anchor_pattern
 from mathics.core.atoms import Integer1, String
@@ -278,7 +279,7 @@ def eval(self, string, evaluation: Evaluation):
             )
             return
 
-        feeder = SingleLineFeeder(string.value)
+        feeder = SingleLineFeeder(string.value, "<SyntaxQ>", ContainerKind.STRING)
         try:
             parser.parse(feeder)
         except SyntaxError:

mathics/builtin/trace.py

@@ -22,16 +22,24 @@
 from time import time
 from typing import Callable
 
+import mathics_scanner.location
+
 import mathics.eval.tracing
 from mathics.core.attributes import A_HOLD_ALL, A_HOLD_ALL_COMPLETE, A_PROTECTED
-from mathics.core.builtin import Builtin
+from mathics.core.builtin import Builtin, Predefined
 from mathics.core.convert.python import from_bool, from_python
 from mathics.core.definitions import Definitions
 from mathics.core.evaluation import Evaluation
 from mathics.core.expression import Expression
 from mathics.core.list import ListExpression
 from mathics.core.rules import FunctionApplyRule
-from mathics.core.symbols import SymbolFalse, SymbolNull, SymbolTrue, strip_context
+from mathics.core.symbols import (
+    Symbol,
+    SymbolFalse,
+    SymbolNull,
+    SymbolTrue,
+    strip_context,
+)
 
 
 def traced_apply_function(
@@ -493,3 +501,33 @@ def eval(self, expr: Expression, evaluation: Evaluation):
         else:
             result = expr.evaluate(evaluation)
         return ListExpression(result, profile_result)
+
+
+class TrackLocations(Predefined):
+    r"""## <url>:TrackLocations native symbol:</url>
+
+    <dl>
+      <dt>'$TrackLocations'
+      <dd>specifies whether we should track \
+      source-text location information during evaluation. This \
+      can be helpful in debugging when there is a failure.
+    </dl>
+    """
+
+    name = "$TrackLocations"
+    messages = {"bool": "`1` should be True or False."}
+
+    summary_text = "track source-text locations in evaluation"
+
+    def evaluate(self, evaluation: Evaluation) -> Symbol:
+        print(mathics_scanner.location.MATHICS3_PATHS)
+        return from_bool(mathics_scanner.location.TRACK_LOCATIONS)
+
+    def eval_set(self, value, evaluation):
+        """Set[$TrackLocations, value_]"""
+        if value is SymbolTrue or value is SymbolFalse:
+            evaluation.definitions.set_ownvalue("System`$TrackLocations", value)
+            mathics.core.parser.parser.TRACK_LOCATIONS = value.to_python()
+        else:
+            evaluation.message("$TrackLocations", "bool", value)
+        return value

mathics/core/builtin.py

@@ -484,7 +484,15 @@ def get_functions(self, prefix="eval", is_pymodule=False):
                 definition_class = (
                     PyMathicsDefinitions() if is_pymodule else SystemDefinitions()
                 )
-                pattern = parse_builtin_rule(pattern, definition_class)
+
+                # Passing the function parameter is in a way
+                # redundant, because creating FunctionApplyRule has
+                # access to the function and sets the postion this
+                # way. But revised afte the dust has settled and
+                # we have a very good idea of what is desirable and useful.
+                pattern = parse_builtin_rule(
+                    pattern, definition_class, location=function
+                )
                 if unavailable_function:
                     function = unavailable_function
                 if attrs:

mathics/core/expression.py

@@ -4,6 +4,7 @@
 import math
 from bisect import bisect_left
 from itertools import chain
+from types import MethodType
 from typing import (
     Any,
     Callable,
@@ -18,6 +19,7 @@
 )
 
 import sympy
+from mathics_scanner.location import SourceRange, SourceRange2
 
 from mathics.core.atoms import Integer1, String
 from mathics.core.attributes import (
@@ -275,6 +277,7 @@ class Expression(BaseElement, NumericOperators, EvalMixin):
     elements_properties: Optional[ElementsProperties]
     options: Optional[Dict[str, Any]]
     pattern_sequence: bool
+    location: Optional[Union[SourceRange, SourceRange2, MethodType]]
 
     def __init__(
         self,
@@ -300,6 +303,7 @@ def __init__(
 
         self._sequences = None
         self._cache = None
+        self.location = None
 
         # self.copy creates this
         self.original: Optional[Expression] = None
@@ -1250,6 +1254,9 @@ def rest_range(indices):
                 head, *elements, elements_properties=self.elements_properties
             )
 
+        if hasattr(self, "location") and self.location is not None:
+            new.location = self.location
+
         # Step 3: Now, process the attributes of head
         # If there are sequence, flatten them if the attributes allow it.
         if (

mathics/core/parser/ast.py

@@ -23,14 +23,15 @@ class Node:
     expression's leaves and a non-leaf nodes.
     """
 
-    def __init__(self, head, *children):
+    def __init__(self, head, *children, location=None):
         if isinstance(head, Node):
             self.head = head
         else:
             self.head = Symbol(head)
         self.value = None
         self.children = list(children)
         self.parenthesised = False
+        self.location = location
 
     def get_head_name(self):
         if isinstance(self.head, Symbol):
@@ -68,11 +69,12 @@ class Atom(Node):
     their own. You can however compare Atoms for equality.
     """
 
-    def __init__(self, value):
+    def __init__(self, value, location=None):
         self.head = Symbol(self.__class__.__name__)
         self.value = value
         self.children = []
         self.parenthesised = False
+        self.location = location
 
     def __repr__(self):
         return "%s[%s]" % (self.head, self.value)
@@ -90,7 +92,13 @@ class Number(Atom):
     """
 
     def __init__(
-        self, value: str, sign: int = 1, base: int = 10, suffix=None, exp: int = 0
+        self,
+        value: str,
+        sign: int = 1,
+        base: int = 10,
+        suffix=None,
+        exp: int = 0,
+        location=None,
     ):
         assert isinstance(value, str)
         assert sign in (-1, 1)
@@ -104,6 +112,7 @@ def __init__(
         self.base = base
         self.suffix = suffix
         self.exp = exp
+        self.location = location
 
     def __repr__(self):
         result = self.value
@@ -132,10 +141,11 @@ class Symbol(Atom):
     are unique as they are say in Lisp, or Python.
     """
 
-    def __init__(self, value: str, context: Optional[str] = "System"):
+    def __init__(self, value: str, context: Optional[str] = "System", location=None):
         self.context = context
         self.value = value
         self.children = []
+        self.location = location
 
     # avoids recursive definition
     @property

mathics/core/parser/feed.py

@@ -21,12 +21,12 @@ def send_messages(self, evaluation) -> list:
 
 
 class MathicsSingleLineFeeder(SingleLineFeeder, MathicsLineFeeder):
-    "A feeder that feeds lines from an open ``File`` object"
+    "A feeder that feeds lines from an open location container object"
 
 
 class MathicsFileLineFeeder(FileLineFeeder, MathicsLineFeeder):
     "A feeder that feeds lines from an open ``File`` object"
 
 
 class MathicsMultiLineFeeder(MultiLineFeeder, MathicsLineFeeder):
-    "A feeder that feeds lines from an open ``File`` object"
+    "A feeder that feeds lines from an open contianer object"