diff --git a/.editorconfig b/.editorconfig index 88af77a0f..316758e94 100644 --- a/.editorconfig +++ b/.editorconfig @@ -1,4 +1,4 @@ -# THis is an EditorConfig file +# This is an EditorConfig file # https://EditorConfig.org root = true diff --git a/.github/workflows/consistency-checks.yml b/.github/workflows/consistency-checks.yml index af01aa365..2d416d1ab 100644 --- a/.github/workflows/consistency-checks.yml +++ b/.github/workflows/consistency-checks.yml @@ -22,13 +22,9 @@ jobs: run: | sudo apt update -qq && sudo apt install llvm-dev remake python -m pip install --upgrade pip - pip install -e . # We can comment out after next Mathics-Scanner release - # python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full] - # git clone https://github.com/Mathics3/mathics-scanner.git - # cd mathics-scanner/ - # pip install -e . - # cd .. + python -m pip install -e git+https://github.com/Mathics3/mathics-scanner@mini-tweaks#egg=Mathics-Scanner[full] + pip install -e . - name: Install Mathics with minimum dependencies run: | diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 5bdcf7d0e..78cee27be 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -33,14 +33,10 @@ jobs: cd stopit/ pip install -e . cd .. - python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full] # We can comment out after next Mathics-Scanner release - git clone --depth 1 https://github.com/Mathics3/mathics-scanner.git - # git clone --single-branch --branch operator-refactor-part1.5 https://github.com/Mathics3/mathics-scanner.git - cd mathics-scanner/ - pip install -e . - cd .. # python -m pip install Mathics-Scanner[full] + python -m pip install -e git+https://github.com/Mathics3/mathics-scanner@mini-tweaks#egg=Mathics-Scanner[full] + pip install -e . remake -x develop-full - name: Test Mathics3 run: | diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 22d56bef5..10aa3b1c6 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -22,21 +22,17 @@ jobs: run: | sudo apt update -qq && sudo apt install llvm-dev remake python -m pip install --upgrade pip - pip install -e . - # python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full] - # We can comment out after next Mathics-Scanner release - # git clone --depth 1 https://github.com/Mathics3/mathics-scanner.git - # cd mathics-scanner/ - # pip install -e . - # cd .. - - name: Install Mathics with minimum dependencies - run: | - make develop - name: Run mypy run: | pip install mypy==1.13 sympy==1.12 - git clone --depth 1 https://github.com/Mathics3/mathics-scanner.git + # Adjust below for right branch + git clone --depth 1 --branch mini-tweaks https://github.com/Mathics3/mathics-scanner.git + cd mathics-scanner/ + pip install -e . + bash ./admin-tools/make-JSON-tables.sh + pip install -e . + cd .. touch ./mathics-scanner/mathics_scanner/py.typed - pip install ./mathics-scanner/ - mypy --install-types --non-interactive mathics + make develop + mypy --install-types --ignore-missing-imports --non-interactive mathics diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 596688775..d7cc0c2b9 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -25,13 +25,7 @@ jobs: run: | python -m pip install --upgrade pip # We can comment out after next Mathics-Scanner release - python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full] - git clone --depth 1 https://github.com/Mathics3/mathics-scanner.git - # git clone --single-branch --branch operator-refactor-part1.5 https://github.com/Mathics3/mathics-scanner.git - cd src/mathics-scanner/ - pip install -e . - python -m mathics_scanner.generate.build_tables - cd ../.. + python -m pip install -e git+https://github.com/Mathics3/mathics-scanner@mini-tweaks#egg=Mathics-Scanner[full] - name: Run Mathics3 Combinatorica tests run: | git submodule init diff --git a/.github/workflows/pyodide.yml b/.github/workflows/pyodide.yml index eb330cda7..80f31bed3 100644 --- a/.github/workflows/pyodide.yml +++ b/.github/workflows/pyodide.yml @@ -55,9 +55,7 @@ jobs: pip install "setuptools>=70.0.0" PyYAML click packaging pytest # We can comment out after next Mathics-Scanner release - # git clone --depth 1 https://github.com/Mathics3/mathics-scanner.git - # git clone --single-branch --branch operator-refactor-part1.5 https://github.com/Mathics3/mathics-scanner.git - # cd mathics-scanner/ + python -m pip install --no-build-isolation -e git+https://github.com/Mathics3/mathics-scanner@mini-tweaks#egg=Mathics-Scanner # pip install --no-build-isolation -e . # cd .. diff --git a/.github/workflows/ubuntu-cython.yml b/.github/workflows/ubuntu-cython.yml index 6eda26f41..9ed38f2bb 100644 --- a/.github/workflows/ubuntu-cython.yml +++ b/.github/workflows/ubuntu-cython.yml @@ -30,10 +30,7 @@ jobs: pip install -e . cd .. # We can comment out after next Mathics-Scanner release - # python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full] - # git clone --depth 1 https://github.com/Mathics3/mathics-scanner.git - # git clone --single-branch --branch operator-refactor-part1.5 https://github.com/Mathics3/mathics-scanner.git - # cd mathics-scanner/ + python -m pip install -e git+https://github.com/Mathics3/mathics-scanner@mini-tweaks#egg=Mathics-Scanner[full] pip install -e . cd .. diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index d886d5b64..4f9b4fb54 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -30,15 +30,9 @@ jobs: pip install -e . cd .. # We can comment out after next Mathics-Scanner release - # python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full] - # git clone --depth 1 https://github.com/Mathics3/mathics-scanner.git - # git clone --single-branch --branch operator-refactor-part1.5 https://github.com/Mathics3/mathics-scanner.git - # cd mathics-scanner/ - # pip install -e . - # python -m mathics_scanner.generate.build_tables - # cd .. - - python -m pip install Mathics-Scanner[full] + # python -m pip install Mathics-Scanner[full] + python -m pip install -e git+https://github.com/Mathics3/mathics-scanner@mini-tweaks#egg=Mathics-Scanner[full] + pip install -e . remake -x develop-full - name: Test Mathics run: | diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index ad2ffffae..4b8ad1925 100755 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -39,13 +39,7 @@ jobs: pip install -e . cd .. # We can comment out after next Mathics-Scanner release - # python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full] - # git clone --depth 1 https://github.com/Mathics3/mathics-scanner.git - # git clone --single-branch --branch operator-refactor-part1.5 https://github.com/Mathics3/mathics-scanner.git - # cd mathics-scanner - # pip install -e . - # python -m mathics_scanner.generate.build_tables - # cd .. + python -m pip install -e git+https://github.com/Mathics3/mathics-scanner@mini-tweaks#egg=Mathics-Scanner[full] pip install -e . # python -m pip install Mathics-Scanner[full] diff --git a/mathics/builtin/atomic/strings.py b/mathics/builtin/atomic/strings.py index 6762c06b0..d4fff9fc1 100644 --- a/mathics/builtin/atomic/strings.py +++ b/mathics/builtin/atomic/strings.py @@ -11,7 +11,7 @@ from heapq import heappop, heappush from typing import Any, List -from mathics_scanner import TranslateError +from mathics_scanner.errors import TranslateError, TranslateErrorNew from mathics.core.atoms import Integer, Integer0, Integer1, String from mathics.core.attributes import A_LISTABLE, A_PROTECTED @@ -20,7 +20,9 @@ from mathics.core.evaluation import Evaluation from mathics.core.expression import Expression from mathics.core.list import ListExpression -from mathics.core.parser import MathicsFileLineFeeder, parse +from mathics.core.parser import MathicsFileLineFeeder +from mathics.core.parser.convert import convert +from mathics.core.parser.util import parser from mathics.core.systemsymbols import ( SymbolFailed, SymbolInputForm, @@ -788,7 +790,7 @@ class ToExpression(Builtin): def eval(self, seq, evaluation: Evaluation): "ToExpression[seq__]" - # Organise Arguments + # From `seq`, extract `inp`, `form`, and `head`. py_seq = seq.get_sequence() if len(py_seq) == 1: (inp, form, head) = (py_seq[0], SymbolInputForm, None) @@ -808,6 +810,7 @@ def eval(self, seq, evaluation: Evaluation): ) return + result = None # Apply the different forms if form is SymbolInputForm: if isinstance(inp, String): @@ -819,13 +822,14 @@ def eval(self, seq, evaluation: Evaluation): feeder = MathicsFileLineFeeder(f) while not feeder.empty(): try: - query = parse(evaluation.definitions, feeder) - except TranslateError: + ast = parser.parse(feeder) + except (TranslateError, TranslateErrorNew): return SymbolFailed finally: feeder.send_messages(evaluation) - if query is None: # blank line / comment + if ast is None: # blank line / comment continue + query = convert(ast, evaluation.definitions) result = query.evaluate(evaluation) else: @@ -835,8 +839,8 @@ def eval(self, seq, evaluation: Evaluation): return # Apply head if present - if head is not None: - result = Expression(head, result).evaluate(evaluation) + if head is not None and result is not None: + return Expression(head, result).evaluate(evaluation) return result diff --git a/mathics/builtin/messages.py b/mathics/builtin/messages.py index 779831201..09b82b98e 100644 --- a/mathics/builtin/messages.py +++ b/mathics/builtin/messages.py @@ -560,9 +560,11 @@ def get_msg_list(expr): evaluation.set_quiet_messages(old_quiet_messages) +# Consider removing. If this was this added just to test some expressions, +# this should be done in pytests instead. class Syntax(Builtin): r""" - :WMA link:https://reference.wolfram.com/language/ref/Syntax.html + :WMA link:https://reference.wolfram.com/language/guide/Syntax.html
'Syntax' @@ -570,16 +572,16 @@ class Syntax(Builtin):
>> 1 + - : Incomplete expression; more input is needed (line 1 of ""). + : Incomplete expression; more input is needed (line 1 of ""). >> Sin[1) - : "Sin[1" cannot be followed by ")" (line 1 of ""). + : "Sin[1" cannot be followed by ")" (line 1 of ""). >> ^ 2 - : Expression cannot begin with "^ 2" (line 1 of ""). + : Expression cannot begin with "^ 2" (line 1 of ""). >> 1.5`` - : "1.5`" cannot be followed by "`" (line 1 of ""). + : "1.5`" cannot be followed by "`" (line 1 of ""). """ # Extension: WMA does not provide lineno and filename in its error messages diff --git a/mathics/builtin/testing_expressions/string_tests.py b/mathics/builtin/testing_expressions/string_tests.py index 20c6eff84..1e1cccc90 100644 --- a/mathics/builtin/testing_expressions/string_tests.py +++ b/mathics/builtin/testing_expressions/string_tests.py @@ -4,7 +4,7 @@ import re -from mathics_scanner import SingleLineFeeder, TranslateError +from mathics_scanner import SingleLineFeeder, TranslateError, TranslateErrorNew from mathics.builtin.atomic.strings import anchor_pattern from mathics.core.atoms import Integer1, String @@ -13,7 +13,7 @@ from mathics.core.convert.regex import to_regex from mathics.core.evaluation import Evaluation from mathics.core.expression import Expression -from mathics.core.parser.util import parse +from mathics.core.parser.util import parser from mathics.core.symbols import Symbol, SymbolFalse, SymbolTrue from mathics.core.systemsymbols import SymbolStringExpression, SymbolStringMatchQ from mathics.eval.strings import eval_StringContainsQ @@ -280,8 +280,8 @@ def eval(self, string, evaluation: Evaluation): feeder = SingleLineFeeder(string.value) try: - parse(evaluation.definitions, feeder) - except TranslateError: + parser.parse(feeder) + except (TranslateError, TranslateErrorNew): return SymbolFalse else: return SymbolTrue diff --git a/mathics/core/convert/sympy.py b/mathics/core/convert/sympy.py index 59994ff3c..89a36ff73 100644 --- a/mathics/core/convert/sympy.py +++ b/mathics/core/convert/sympy.py @@ -129,7 +129,7 @@ def to_sympy_matrix(data, **kwargs) -> Optional[sympy.MutableDenseMatrix]: return None -class SympyExpression(BasicSympy): +class SympyExpression(sympy.Expr): """A Sympy expression with an associated Mathics expression""" is_Function = True diff --git a/mathics/core/evaluation.py b/mathics/core/evaluation.py index e50c76b3d..aef6a793f 100644 --- a/mathics/core/evaluation.py +++ b/mathics/core/evaluation.py @@ -6,7 +6,7 @@ from abc import ABC from typing import Any, Callable, Dict, List, Optional, Tuple, Union, overload -from mathics_scanner import TranslateError +from mathics_scanner.errors import IncompleteSyntaxError, InvalidSyntaxError, ScanError from mathics import settings from mathics.core.atoms import Integer, String @@ -161,16 +161,20 @@ def parse_feeder_returning_code_and_messages(self, feeder) -> tuple: Parse a single expression from feeder, print the messages it produces and return the result, the source code for this and evaluated messages created in evaluation. + + If there was a TranslateError, the source code returned is "" and the result is None. """ from mathics.core.parser.util import parse_returning_code try: result, source_code = parse_returning_code(self.definitions, feeder) - except TranslateError: + except (InvalidSyntaxError, IncompleteSyntaxError, ScanError): + result = None + source_code = "" + + if result is None: self.recursion_depth = 0 self.stopped = False - source_code = "" - result = None messages = feeder.send_messages(self) return result, source_code, messages diff --git a/mathics/core/parser/README.md b/mathics/core/parser/README.md index 2128fda69..d9bf7035e 100644 --- a/mathics/core/parser/README.md +++ b/mathics/core/parser/README.md @@ -90,8 +90,8 @@ def parse_binary(self, expr1, token, expr1_precedence: int) -> Node: # handle nonassoc operators if tag in nonassoc_binary_ops and expr1.get_head_name() == tag and not expr1.parenthesised: - self.tokeniser.sntx_message(token.pos) - raise InvalidSyntaxError() + tag, pre_error, post_error = self.tokeniser.sntx_message(token.pos) + raise InvalidSyntaxError(tag, pre_error, post_error) result = Node(tag, expr1, expr2) # construct the result: `BINARY[expr1, expr2]` diff --git a/mathics/core/parser/feed.py b/mathics/core/parser/feed.py index 0661def7d..fba41e37f 100644 --- a/mathics/core/parser/feed.py +++ b/mathics/core/parser/feed.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +from typing import List + from mathics_scanner import ( FileLineFeeder, LineFeeder, @@ -8,6 +10,8 @@ class MathicsLineFeeder(LineFeeder): + messages: List[str] + def send_messages(self, evaluation) -> list: evaluated_messages = [] for message in self.messages: diff --git a/mathics/core/parser/parser.py b/mathics/core/parser/parser.py index 6cc017e30..6146faae8 100644 --- a/mathics/core/parser/parser.py +++ b/mathics/core/parser/parser.py @@ -10,7 +10,7 @@ import string from typing import Optional, Union -from mathics_scanner import InvalidSyntaxError, TranslateError +from mathics_scanner.errors import InvalidSyntaxError, TranslateError, TranslateErrorNew from mathics_scanner.tokeniser import Token, Tokeniser, is_symbol_name from mathics.core.convert.op import builtin_constants @@ -122,11 +122,11 @@ def expect(self, expected_tag: str): if token.tag == expected_tag: self.consume() else: - self.tokeniser.sntx_message(token.pos) - raise InvalidSyntaxError() + tag, pre_error, post_error = self.tokeniser.sntx_message(token.pos) + raise InvalidSyntaxError(tag, pre_error, post_error) - def incomplete(self, pos: int): - self.tokeniser.incomplete() + def get_more_input(self, pos: int): + self.tokeniser.get_more_input() self.backtrack(pos) @property @@ -148,7 +148,7 @@ def next_noend(self) -> Token: token = self.next() if token.tag != "END": return token - self.incomplete(token.pos) + self.get_more_input(token.pos) def parse(self, feeder) -> Optional[Node]: """ @@ -225,8 +225,8 @@ def parse_binary_operator( and expr1.get_head_name() == tag and not expr1.parenthesised ): - self.tokeniser.sntx_message(token.pos) - raise InvalidSyntaxError() + tag, pre_error, post_error = self.tokeniser.sntx_message(token.pos) + raise InvalidSyntaxError(tag, pre_error, post_error) result = Node(tag, expr1, expr2) @@ -266,7 +266,7 @@ def parse_box_expr(self, precedence: int) -> Union[String, Node]: elif tag in ("OtherscriptBox", "RightRowBox"): break elif tag == "END": - self.incomplete(token.pos) + self.get_more_input(token.pos) elif result is None and tag != "END": self.consume() # TODO: handle non-box expressions inside RowBox @@ -446,8 +446,10 @@ def parse_expr(self, precedence: int) -> Optional[Node]: if self.is_inside_rowbox: break else: - self.tokeniser.sntx_message(token.pos) - raise InvalidSyntaxError() + tag, pre_error, post_error = self.tokeniser.sntx_message( + token.pos + ) + raise InvalidSyntaxError(tag, pre_error, post_error) else: token = self.next() @@ -508,8 +510,8 @@ def parse_p(self): elif self.is_inside_rowbox: return None else: - self.tokeniser.sntx_message(token.pos) - raise InvalidSyntaxError() + tag, pre_error, post_error = self.tokeniser.sntx_message(token.pos) + raise InvalidSyntaxError(tag, pre_error, post_error) def parse_postfix( self, expr1, token: Token, expr1_precedence: int @@ -805,12 +807,12 @@ def e_MessageName(self, expr1, token: Token, p: int) -> Node: elif token.tag == "String": element = self.p_String(token) else: - self.tokeniser.sntx_message(token.pos) - raise InvalidSyntaxError() + tag, pre, post = self.tokeniser.sntx_message(token.pos) + raise InvalidSyntaxError(tag, pre, post) elements.append(element) return Node("MessageName", *elements) - def e_Minus(self, expr1, token: Token, p: int) -> Optional[Node]: + def e_Minus(self, expr1, _: Token, p: int) -> Optional[Node]: q = left_binary_operators["Subtract"] if q < p: return None @@ -822,7 +824,7 @@ def e_Minus(self, expr1, token: Token, p: int) -> Optional[Node]: expr2 = Node("Times", NumberM1, expr2).flatten() return Node("Plus", expr1, expr2).flatten() - def e_Prefix(self, expr1, token: Token, expr1_precedence: int) -> Optional[Node]: + def e_Prefix(self, expr1, _: Token, expr1_precedence: int) -> Optional[Node]: """ Used to parse: expr1 @ expr2 @@ -966,7 +968,7 @@ def e_Semicolon(self, expr1, token: Token, expr1_precedence: int) -> Optional[No # XXX look for next expr otherwise backtrack try: expr2 = self.parse_expr(operator_precedence + 1) - except TranslateError: + except (TranslateError, TranslateErrorNew): self.backtrack(pos) self.feeder.messages = messages expr2 = NullSymbol @@ -993,7 +995,7 @@ def e_Span(self, expr1, token: Token, p) -> Optional[Node]: messages = list(self.feeder.messages) try: expr2 = self.parse_expr(q + 1) - except TranslateError: + except (TranslateError, TranslateErrorNew): expr2 = Symbol("All") self.backtrack(token.pos) self.feeder.messages = messages @@ -1004,7 +1006,7 @@ def e_Span(self, expr1, token: Token, p) -> Optional[Node]: try: expr3 = self.parse_expr(q + 1) return Node("Span", expr1, expr2, expr3) - except TranslateError: + except (TranslateError, TranslateErrorNew): self.backtrack(token.pos) self.feeder.messages = messages return Node("Span", expr1, expr2) @@ -1025,8 +1027,8 @@ def e_TagSet(self, expr1, token: Token, p: int) -> Optional[Node]: elif tag == "Unset": head = "TagUnset" else: - self.tokeniser.sntx_message(token.pos) - raise InvalidSyntaxError() + tag, pre_error, post_error = self.tokeniser.sntx_message(token.pos) + raise InvalidSyntaxError(tag, pre_error, post_error) self.consume() if head == "TagUnset": return Node(head, expr1, expr2) @@ -1063,7 +1065,7 @@ def p_Information(self, token: Token) -> Node: q = prefix_operators["Information"] child = self.parse_expr(q) if child.__class__ is not Symbol: - raise InvalidSyntaxError() + return Node("Missing", String("UnknownSymbol"), child) return Node( "Information", child, Node("Rule", Symbol("LongForm"), Symbol("True")) ) @@ -1091,6 +1093,7 @@ def p_LeftRowBox(self, token: Token) -> Union[Node, String]: self.consume() children = [] self.box_depth += 1 + self.tokeniser.is_inside_box = True token = self.next() while token.tag not in ("RightRowBox", "OtherscriptBox"): newnode = self.parse_box_expr(NEVER_ADD_PARENTHESIS) @@ -1105,6 +1108,7 @@ def p_LeftRowBox(self, token: Token) -> Union[Node, String]: result = Node("RowBox", Node("List", *children)) self.expect("RightRowBox") self.box_depth -= 1 + self.tokeniser.is_inside_box = self.box_depth > 0 result.parenthesised = True return result @@ -1169,8 +1173,8 @@ def p_Number(self, token: Token) -> Number: base, s = int(base_parts[0]), base_parts[1] if not 2 <= base <= 36: self.tokeniser.feeder.message("General", "base", base, token.text, 36) - self.tokeniser.sntx_message(token.pos) - raise InvalidSyntaxError() + _, pre_error, post_error = self.tokeniser.sntx_message(token.pos) + raise InvalidSyntaxError("General", "base", pre_error, post_error) # mantissa mantissa_parts = s.split("*^") @@ -1190,8 +1194,8 @@ def p_Number(self, token: Token) -> Number: for i, c in enumerate(s.lower()): if permitted_digits[c] >= base: self.tokeniser.feeder.message("General", "digit", i + 1, s, base) - self.tokeniser.sntx_message(token.pos) - raise InvalidSyntaxError() + _, pre_error, post_error = self.tokeniser.sntx_message(token.pos) + raise InvalidSyntaxError("General", "digit", pre_error, post_error) result = Number(s, sign=sign, base=base, suffix=suffix, exp=exp) self.consume() diff --git a/mathics/core/parser/util.py b/mathics/core/parser/util.py index 505162862..f01f14ede 100644 --- a/mathics/core/parser/util.py +++ b/mathics/core/parser/util.py @@ -1,17 +1,25 @@ -#!/usr/bin/env python3 # -*- coding: utf-8 -*- from typing import Any, FrozenSet, Tuple +from mathics_scanner.errors import ( + IncompleteSyntaxError, + InvalidSyntaxError, + TranslateError, + TranslateErrorNew, +) +from mathics_scanner.feed import LineFeeder + from mathics.core.parser.convert import convert from mathics.core.parser.feed import MathicsSingleLineFeeder from mathics.core.parser.parser import Parser -from mathics.core.symbols import ensure_context +from mathics.core.symbols import Symbol, SymbolNull, ensure_context +from mathics.core.systemsymbols import SymbolFailed parser = Parser() -def parse(definitions, feeder) -> Any: +def parse(definitions, feeder: LineFeeder) -> Any: """ Parse input (from the frontend, -e, input files, ToExpression etc). Look up symbols according to the Definitions instance supplied. @@ -21,7 +29,35 @@ def parse(definitions, feeder) -> Any: return parse_returning_code(definitions, feeder)[0] -def parse_returning_code(definitions, feeder) -> Tuple[Any, str]: +def parse_incrementally_by_line(definitions, feeder: LineFeeder) -> Any: + """Parse input incrementally by line. This is in contrast to parse() or + parser_returning_code(), which parse the *entire* + input which could be many line. + + This routine is called via Read[] which parses by line, possibly + leaving of the input unparsed, depending on whether Read[] + requires more expressions. + + By working incrementally, we may avoid reading lots of input that + is not going to be needed. + + As a result, we do *not* handle exceptions raised. Instead, we leave that for the + eval_Read() routine to handle, so it can ask for another line. + + Feeder must implement the feed and empty methods. + + The result is the AST parsed or syhmbols like $Failed or NullType. Or there can be + an exception raised in parse which filters through this routine. + + """ + + ast = parser.parse(feeder) + if ast is None or isinstance(ast, Symbol): + return ast + return convert(ast, definitions) + + +def parse_returning_code(definitions, feeder: LineFeeder) -> Tuple[Any, str]: """ Parse input (from the frontend, -e, input files, ToExpression etc). Look up symbols according to the Definitions instance supplied. diff --git a/mathics/doc/documentation/1-Manual.mdoc b/mathics/doc/documentation/1-Manual.mdoc index 0363603a0..231aebb29 100644 --- a/mathics/doc/documentation/1-Manual.mdoc +++ b/mathics/doc/documentation/1-Manual.mdoc @@ -919,7 +919,7 @@ For instance, you can override 'MakeBoxes' to format lists in a different way: However, this will not be accepted as input to \Mathics anymore: >> [1 2 3] - : Expression cannot begin with "[1 2 3]" (line 1 of ""). + : Expression cannot begin with "[1 2 3]" (line 1 of ""). >> Clear[MakeBoxes] diff --git a/mathics/docpipeline.py b/mathics/docpipeline.py index c7b16d839..bebe1145f 100644 --- a/mathics/docpipeline.py +++ b/mathics/docpipeline.py @@ -200,6 +200,7 @@ def show_test(self, test: DocTest, index: int, subindex: int): def test_case( test: DocTest, + src_name: str, test_pipeline: DocTestPipeline, fail: Callable, ) -> bool: @@ -213,7 +214,7 @@ def test_case( test_parameters = test_pipeline.parameters try: time_start = datetime.now() - result = test_pipeline.session.evaluate_as_in_cli(test.test, src_name="") + result = test_pipeline.session.evaluate_as_in_cli(test.test, src_name=src_name) out = result.out result = result.result except Exception as exc: @@ -428,6 +429,7 @@ def test_section_in_chapter( continue section_name_for_print = test_status.section_name_for_print(doctest) test_status.show_section(doctest) + assert doctest.key is not None key = list(doctest.key)[1:-1] if key != test_status.prev_key: index = 1 @@ -451,6 +453,7 @@ def fail_message(why): success = test_case( doctest, + f"", test_pipeline, fail=fail_message, ) @@ -643,6 +646,7 @@ def test_sections( # show_test_summary(test_pipeline, "sections", section_names) # return + assert section_names is not None show_test_summary(test_pipeline, "sections", section_names) return diff --git a/mathics/eval/files_io/files.py b/mathics/eval/files_io/files.py index b9e1bf44a..0d63bfcf6 100644 --- a/mathics/eval/files_io/files.py +++ b/mathics/eval/files_io/files.py @@ -18,7 +18,8 @@ from mathics.core.convert.python import from_python from mathics.core.evaluation import Evaluation from mathics.core.expression import BaseElement, Expression -from mathics.core.parser import MathicsFileLineFeeder, MathicsMultiLineFeeder, parse +from mathics.core.parser import MathicsFileLineFeeder, MathicsMultiLineFeeder +from mathics.core.parser.util import parse_incrementally_by_line from mathics.core.streams import path_search, stream_manager from mathics.core.symbols import Symbol, SymbolNull from mathics.core.systemsymbols import ( @@ -264,14 +265,18 @@ def eval_Read( result.append(tmp) elif typ in (SymbolExpression, SymbolHoldExpression): tmp = next(read_record) + assert isinstance(tmp, str) while True: try: feeder = MathicsMultiLineFeeder(tmp) - expr = parse(evaluation.definitions, feeder) + expr = parse_incrementally_by_line( + evaluation.definitions, feeder + ) break except (IncompleteSyntaxError, InvalidSyntaxError): try: nextline = next(read_record) + assert isinstance(nextline, str) tmp = tmp + "\n" + nextline except EOFError: expr = SymbolEndOfFile diff --git a/pyproject.toml b/pyproject.toml index fdf9d9b96..12f460fd9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ build-backend = "setuptools.build_meta" [project] description = "A general-purpose computer algebra system." dependencies = [ - "Mathics-Scanner >= 1.4.1", + "Mathics-Scanner > 1.4.1", "mpmath>=1.2.0", "numpy<2.3", "palettable", diff --git a/test/builtin/atomic/test_strings2.py b/test/builtin/atomic/test_strings2.py index 3196b8dc4..05ba2f7af 100644 --- a/test/builtin/atomic/test_strings2.py +++ b/test/builtin/atomic/test_strings2.py @@ -44,11 +44,11 @@ def test_string_split(): "{{11, 12, 13}, {21, 22, 23}, {31, 32, 33}}", ), ( - 'StringSplit["A tree, an apple, four pears. And more: two sacks", RegularExpression["\\W+"]]', + r'StringSplit["A tree, an apple, four pears. And more: two sacks", RegularExpression["\\W+"]]', "{A, tree, an, apple, four, pears, And, more, two, sacks}", ), ( - 'StringSplit["primes: 2 two 3 three 5 five ...", Whitespace ~~ RegularExpression["\\d"] ~~ Whitespace]', + r'StringSplit["primes: 2 two 3 three 5 five ...", Whitespace ~~ RegularExpression["\\d"] ~~ Whitespace]', "{primes:, two, three, five ...}", ), ('StringSplit["a-b:c-d:e-f-g", {":", "-"}]', "{a, b, c, d, e, f, g}"), diff --git a/test/core/parser/test_parser.py b/test/core/parser/test_parser.py index 398e2217c..0f65a5464 100644 --- a/test/core/parser/test_parser.py +++ b/test/core/parser/test_parser.py @@ -630,7 +630,7 @@ def testInequality(self): def testInformation(self): self.check("??a", "Information[a, LongForm -> True]") self.check("a ?? b", "a Information[b, LongForm -> True]") - self.invalid_error("a ?? + b") + self.check("a ?? + b", 'Times[a, Missing["UnknownSymbol", Plus[b]]]') self.check("a + ?? b", "a + Information[b, LongForm -> True]") self.check("??a + b", "Information[a, LongForm -> True] + b") self.check("??a * b", "Information[a, Rule[LongForm, True]]*b")