Track scanner (partial) refactor changes

.editorconfig

@@ -1,4 +1,4 @@
-# THis is an EditorConfig file
+# This is an EditorConfig file
 # https://EditorConfig.org
 
 root = true

.github/workflows/consistency-checks.yml

@@ -22,13 +22,9 @@ jobs:
       run: |
         sudo apt update -qq && sudo apt install llvm-dev remake
         python -m pip install --upgrade pip
-        pip install -e .
         # We can comment out after next Mathics-Scanner release
-        # python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full]
-        # git clone https://github.com/Mathics3/mathics-scanner.git
-        # cd mathics-scanner/
-        # pip install -e .
-        # cd ..
+        python -m pip install -e git+https://github.com/Mathics3/mathics-scanner@mini-tweaks#egg=Mathics-Scanner[full]
+        pip install -e .
 
     - name: Install Mathics with minimum dependencies
       run: |

.github/workflows/macos.yml

@@ -33,14 +33,10 @@ jobs:
         cd stopit/
         pip install -e .
         cd ..
-        python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full]
         # We can comment out after next Mathics-Scanner release
-        git clone --depth 1 https://github.com/Mathics3/mathics-scanner.git
-        # git clone --single-branch --branch operator-refactor-part1.5 https://github.com/Mathics3/mathics-scanner.git
-        cd mathics-scanner/
-        pip install -e .
-        cd ..
         # python -m pip install Mathics-Scanner[full]
+        python -m pip install -e git+https://github.com/Mathics3/mathics-scanner@mini-tweaks#egg=Mathics-Scanner[full]
+        pip install -e .
         remake -x develop-full
     - name: Test Mathics3
       run: |

.github/workflows/mypy.yml

@@ -22,21 +22,17 @@ jobs:
       run: |
         sudo apt update -qq && sudo apt install llvm-dev remake
         python -m pip install --upgrade pip
-        pip install -e .
-        # python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full]
-        # We can comment out after next Mathics-Scanner release
-        # git clone --depth 1 https://github.com/Mathics3/mathics-scanner.git
-        # cd mathics-scanner/
-        # pip install -e .
-        # cd ..
 
-    - name: Install Mathics with minimum dependencies
-      run: |
-        make develop
     - name: Run mypy
       run: |
         pip install mypy==1.13 sympy==1.12
-        git clone --depth 1 https://github.com/Mathics3/mathics-scanner.git
+        # Adjust below for right branch
+        git clone --depth 1 --branch mini-tweaks https://github.com/Mathics3/mathics-scanner.git
+        cd mathics-scanner/
+        pip install -e .
+        bash ./admin-tools/make-JSON-tables.sh
+        pip install -e .
+        cd ..
         touch ./mathics-scanner/mathics_scanner/py.typed
-        pip install ./mathics-scanner/
-        mypy --install-types --non-interactive mathics
+        make develop
+        mypy --install-types --ignore-missing-imports --non-interactive mathics

.github/workflows/packages.yml

@@ -25,13 +25,7 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         # We can comment out after next Mathics-Scanner release
-        python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full]
-        git clone --depth 1 https://github.com/Mathics3/mathics-scanner.git
-        # git clone --single-branch --branch operator-refactor-part1.5 https://github.com/Mathics3/mathics-scanner.git
-        cd src/mathics-scanner/
-        pip install -e .
-        python -m mathics_scanner.generate.build_tables
-        cd ../..
+        python -m pip install -e git+https://github.com/Mathics3/mathics-scanner@mini-tweaks#egg=Mathics-Scanner[full]
     - name: Run Mathics3 Combinatorica tests
       run: |
         git submodule init

.github/workflows/pyodide.yml

@@ -55,9 +55,7 @@ jobs:
           pip install "setuptools>=70.0.0" PyYAML click packaging pytest
 
           # We can comment out after next Mathics-Scanner release
-          # git clone --depth 1 https://github.com/Mathics3/mathics-scanner.git
-          # git clone --single-branch --branch operator-refactor-part1.5 https://github.com/Mathics3/mathics-scanner.git
-          # cd mathics-scanner/
+          python -m pip install --no-build-isolation -e git+https://github.com/Mathics3/mathics-scanner@mini-tweaks#egg=Mathics-Scanner
           # pip install --no-build-isolation -e .
           # cd ..
 

.github/workflows/ubuntu-cython.yml

@@ -30,10 +30,7 @@ jobs:
         pip install -e .
         cd ..
         # We can comment out after next Mathics-Scanner release
-        # python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full]
-        # git clone --depth 1 https://github.com/Mathics3/mathics-scanner.git
-        # git clone --single-branch --branch operator-refactor-part1.5 https://github.com/Mathics3/mathics-scanner.git
-        # cd mathics-scanner/
+        python -m pip install -e git+https://github.com/Mathics3/mathics-scanner@mini-tweaks#egg=Mathics-Scanner[full]
         pip install -e .
         cd ..
 

.github/workflows/ubuntu.yml

@@ -30,15 +30,9 @@ jobs:
         pip install -e .
         cd ..
         # We can comment out after next Mathics-Scanner release
-        # python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full]
-        # git clone --depth 1 https://github.com/Mathics3/mathics-scanner.git
-        # git clone --single-branch --branch operator-refactor-part1.5 https://github.com/Mathics3/mathics-scanner.git
-        # cd mathics-scanner/
-        # pip install -e .
-        # python -m mathics_scanner.generate.build_tables
-        # cd ..
-
-        python -m pip install Mathics-Scanner[full]
+        # python -m pip install Mathics-Scanner[full]
+        python -m pip install -e git+https://github.com/Mathics3/mathics-scanner@mini-tweaks#egg=Mathics-Scanner[full]
+        pip install -e .
         remake -x develop-full
     - name: Test Mathics
       run: |

.github/workflows/windows.yml

@@ -39,13 +39,7 @@ jobs:
         pip install -e .
         cd ..
         # We can comment out after next Mathics-Scanner release
-        # python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full]
-        # git clone --depth 1 https://github.com/Mathics3/mathics-scanner.git
-        # git clone --single-branch --branch operator-refactor-part1.5 https://github.com/Mathics3/mathics-scanner.git
-        # cd mathics-scanner
-        # pip install -e .
-        # python -m mathics_scanner.generate.build_tables
-        # cd ..
+        python -m pip install -e git+https://github.com/Mathics3/mathics-scanner@mini-tweaks#egg=Mathics-Scanner[full]
         pip install -e .
 
         # python -m pip install Mathics-Scanner[full]

mathics/builtin/atomic/strings.py

@@ -11,7 +11,7 @@
 from heapq import heappop, heappush
 from typing import Any, List
 
-from mathics_scanner import TranslateError
+from mathics_scanner.errors import TranslateError, TranslateErrorNew
 
 from mathics.core.atoms import Integer, Integer0, Integer1, String
 from mathics.core.attributes import A_LISTABLE, A_PROTECTED
@@ -20,7 +20,9 @@
 from mathics.core.evaluation import Evaluation
 from mathics.core.expression import Expression
 from mathics.core.list import ListExpression
-from mathics.core.parser import MathicsFileLineFeeder, parse
+from mathics.core.parser import MathicsFileLineFeeder
+from mathics.core.parser.convert import convert
+from mathics.core.parser.util import parser
 from mathics.core.systemsymbols import (
     SymbolFailed,
     SymbolInputForm,
@@ -788,7 +790,7 @@ class ToExpression(Builtin):
     def eval(self, seq, evaluation: Evaluation):
         "ToExpression[seq__]"
 
-        # Organise Arguments
+        # From `seq`, extract `inp`, `form`, and `head`.
         py_seq = seq.get_sequence()
         if len(py_seq) == 1:
             (inp, form, head) = (py_seq[0], SymbolInputForm, None)
@@ -808,6 +810,7 @@ def eval(self, seq, evaluation: Evaluation):
             )
             return
 
+        result = None
         # Apply the different forms
         if form is SymbolInputForm:
             if isinstance(inp, String):
@@ -819,13 +822,14 @@ def eval(self, seq, evaluation: Evaluation):
                     feeder = MathicsFileLineFeeder(f)
                     while not feeder.empty():
                         try:
-                            query = parse(evaluation.definitions, feeder)
-                        except TranslateError:
+                            ast = parser.parse(feeder)
+                        except (TranslateError, TranslateErrorNew):
                             return SymbolFailed
                         finally:
                             feeder.send_messages(evaluation)
-                        if query is None:  # blank line / comment
+                        if ast is None:  # blank line / comment
                             continue
+                        query = convert(ast, evaluation.definitions)
                         result = query.evaluate(evaluation)
 
             else:
@@ -835,8 +839,8 @@ def eval(self, seq, evaluation: Evaluation):
             return
 
         # Apply head if present
-        if head is not None:
-            result = Expression(head, result).evaluate(evaluation)
+        if head is not None and result is not None:
+            return Expression(head, result).evaluate(evaluation)
 
         return result
 

mathics/builtin/messages.py

@@ -560,26 +560,28 @@ def get_msg_list(expr):
             evaluation.set_quiet_messages(old_quiet_messages)
 
 
+# Consider removing. If this was this added just to test some expressions,
+# this should be done in pytests instead.
 class Syntax(Builtin):
     r"""
-    <url>:WMA link:https://reference.wolfram.com/language/ref/Syntax.html</url>
+    <url>:WMA link:https://reference.wolfram.com/language/guide/Syntax.html</url>
 
     <dl>
       <dt>'Syntax'
       <dd>is a symbol to which all syntax messages are assigned.
     </dl>
 
     >> 1 +
-     : Incomplete expression; more input is needed (line 1 of "<test>").
+     : Incomplete expression; more input is needed (line 1 of "<test-Syntax-1>").
 
     >> Sin[1)
-     : "Sin[1" cannot be followed by ")" (line 1 of "<test>").
+     : "Sin[1" cannot be followed by ")" (line 1 of "<test-Syntax-2>").
 
     >> ^ 2
-     : Expression cannot begin with "^ 2" (line 1 of "<test>").
+     : Expression cannot begin with "^ 2" (line 1 of "<test-Syntax-3>").
 
     >> 1.5``
-     : "1.5`" cannot be followed by "`" (line 1 of "<test>").
+     : "1.5`" cannot be followed by "`" (line 1 of "<test-Syntax-4>").
     """
 
     # Extension: WMA does not provide lineno and filename in its error messages

mathics/builtin/testing_expressions/string_tests.py

@@ -4,7 +4,7 @@
 
 import re
 
-from mathics_scanner import SingleLineFeeder, TranslateError
+from mathics_scanner import SingleLineFeeder, TranslateError, TranslateErrorNew
 
 from mathics.builtin.atomic.strings import anchor_pattern
 from mathics.core.atoms import Integer1, String
@@ -13,7 +13,7 @@
 from mathics.core.convert.regex import to_regex
 from mathics.core.evaluation import Evaluation
 from mathics.core.expression import Expression
-from mathics.core.parser.util import parse
+from mathics.core.parser.util import parser
 from mathics.core.symbols import Symbol, SymbolFalse, SymbolTrue
 from mathics.core.systemsymbols import SymbolStringExpression, SymbolStringMatchQ
 from mathics.eval.strings import eval_StringContainsQ
@@ -280,8 +280,8 @@ def eval(self, string, evaluation: Evaluation):
 
         feeder = SingleLineFeeder(string.value)
         try:
-            parse(evaluation.definitions, feeder)
-        except TranslateError:
+            parser.parse(feeder)
+        except (TranslateError, TranslateErrorNew):
             return SymbolFalse
         else:
             return SymbolTrue

mathics/core/convert/sympy.py

@@ -129,7 +129,7 @@ def to_sympy_matrix(data, **kwargs) -> Optional[sympy.MutableDenseMatrix]:
         return None
 
 
-class SympyExpression(BasicSympy):
+class SympyExpression(sympy.Expr):
     """A Sympy expression with an associated Mathics expression"""
 
     is_Function = True

mathics/core/evaluation.py

@@ -6,7 +6,7 @@
 from abc import ABC
 from typing import Any, Callable, Dict, List, Optional, Tuple, Union, overload
 
-from mathics_scanner import TranslateError
+from mathics_scanner.errors import IncompleteSyntaxError, InvalidSyntaxError, ScanError
 
 from mathics import settings
 from mathics.core.atoms import Integer, String
@@ -161,16 +161,20 @@ def parse_feeder_returning_code_and_messages(self, feeder) -> tuple:
         Parse a single expression from feeder, print the messages it produces and
         return the result, the source code for this and evaluated
         messages created in evaluation.
+
+        If there was a TranslateError, the source code returned is "" and the result is None.
         """
         from mathics.core.parser.util import parse_returning_code
 
         try:
             result, source_code = parse_returning_code(self.definitions, feeder)
-        except TranslateError:
+        except (InvalidSyntaxError, IncompleteSyntaxError, ScanError):
+            result = None
+            source_code = ""
+
+        if result is None:
             self.recursion_depth = 0
             self.stopped = False
-            source_code = ""
-            result = None
         messages = feeder.send_messages(self)
         return result, source_code, messages
 

mathics/core/parser/README.md

@@ -90,8 +90,8 @@ def parse_binary(self, expr1, token, expr1_precedence: int) -> Node:
 
                                             # handle nonassoc operators
     if tag in nonassoc_binary_ops and expr1.get_head_name() == tag and not expr1.parenthesised:
-        self.tokeniser.sntx_message(token.pos)
-        raise InvalidSyntaxError()
+        tag, pre_error, post_error = self.tokeniser.sntx_message(token.pos)
+        raise InvalidSyntaxError(tag, pre_error, post_error)
 
     result = Node(tag, expr1, expr2)        # construct the result: `BINARY[expr1, expr2]`
 

mathics/core/parser/feed.py

@@ -1,4 +1,6 @@
 # -*- coding: utf-8 -*-
+from typing import List
+
 from mathics_scanner import (
     FileLineFeeder,
     LineFeeder,
@@ -8,6 +10,8 @@
 
 
 class MathicsLineFeeder(LineFeeder):
+    messages: List[str]
+
     def send_messages(self, evaluation) -> list:
         evaluated_messages = []
         for message in self.messages: