From a3520b1b00828ed154c25c27b329629a57176216 Mon Sep 17 00:00:00 2001 From: telamonian Date: Sun, 11 Oct 2020 22:33:06 -0400 Subject: [PATCH 1/9] added literal_eval_index.py with working literal_eval_index function --- .gitignore | 12 +++++++ ndindex/literal_eval_index.py | 63 +++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+) create mode 100644 ndindex/literal_eval_index.py diff --git a/.gitignore b/.gitignore index 3de4f1e8..82e989c9 100644 --- a/.gitignore +++ b/.gitignore @@ -130,3 +130,15 @@ dmypy.json # Pyre type checker .pyre/ + +# jetbrains ide stuff +*.iml +.idea/ + +# vscode ide stuff +*.code-workspace +.history/ +.vscode/ + +# project scratch dir +/scratch/ diff --git a/ndindex/literal_eval_index.py b/ndindex/literal_eval_index.py new file mode 100644 index 00000000..faf79d1b --- /dev/null +++ b/ndindex/literal_eval_index.py @@ -0,0 +1,63 @@ +import ast +import sys + +def literal_eval_slice(node_or_string): + """ + "Safely" (needs validation) evaluate an expression node or a string containing + a (limited subset) of valid numpy index or slice expressions. + """ + if isinstance(node_or_string, str): + node_or_string = ast.parse('dummy[{}]'.format(node_or_string.lstrip(" \t")) , mode='eval') + if isinstance(node_or_string, ast.Expression): + node_or_string = node_or_string.body + if isinstance(node_or_string, ast.Subscript): + node_or_string = node_or_string.slice + + def _raise_malformed_node(node): + raise ValueError(f'malformed node or string: {node!r}') + + # from cpy37, should work until they remove ast.Num (not until cpy310) + def _convert_num(node): + if isinstance(node, ast.Constant): + if isinstance(node.value, (int, float, complex)): + return node.value + elif isinstance(node, ast.Num): + return node.n + raise ValueError('malformed node or string: ' + repr(node)) + def _convert_signed_num(node): + if isinstance(node, ast.UnaryOp) and isinstance(node.op, (ast.UAdd, ast.USub)): + operand = _convert_num(node.operand) + if isinstance(node.op, ast.UAdd): + return + operand + else: + return - operand + return _convert_num(node) + + def _convert(node): + if isinstance(node, ast.Constant): + return node.value + elif isinstance(node, ast.Tuple): + return tuple(map(_convert, node.elts)) + elif isinstance(node, ast.List): + return list(map(_convert, node.elts)) + elif isinstance(node, ast.Slice): + return slice( + _convert(node.lower) if node.lower is not None else None, + _convert(node.upper) if node.upper is not None else None, + _convert(node.step) if node.step is not None else None, + ) + + if sys.version_info.major == 3 and sys.version_info.minor < 9: + if sys.version_info.minor < 8: + # ast.Num was removed (all uses replaced with ast.Constant) in cpy38 + if isinstance(node, ast.Num): + return node.n + + # ast.Index and ast.ExtSlice were removed in cpy39 + if isinstance(node, ast.Index): + return _convert(node.value) + elif isinstance(node, ast.ExtSlice): + return tuple(map(_convert, node.dims)) + + return _convert_signed_num(node) + return _convert(node_or_string) From 35a6b8a295e9edd892524c0248ab06774a88f841 Mon Sep 17 00:00:00 2001 From: telamonian Date: Sun, 11 Oct 2020 22:36:36 -0400 Subject: [PATCH 2/9] removed unnecessary cpy minor version guards from literal_eval_index --- ndindex/literal_eval_index.py | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/ndindex/literal_eval_index.py b/ndindex/literal_eval_index.py index faf79d1b..e9f274e7 100644 --- a/ndindex/literal_eval_index.py +++ b/ndindex/literal_eval_index.py @@ -1,5 +1,4 @@ import ast -import sys def literal_eval_slice(node_or_string): """ @@ -46,18 +45,15 @@ def _convert(node): _convert(node.upper) if node.upper is not None else None, _convert(node.step) if node.step is not None else None, ) - - if sys.version_info.major == 3 and sys.version_info.minor < 9: - if sys.version_info.minor < 8: - # ast.Num was removed (all uses replaced with ast.Constant) in cpy38 - if isinstance(node, ast.Num): - return node.n - - # ast.Index and ast.ExtSlice were removed in cpy39 - if isinstance(node, ast.Index): - return _convert(node.value) - elif isinstance(node, ast.ExtSlice): - return tuple(map(_convert, node.dims)) + elif isinstance(node, ast.Num): + # ast.Num was removed from ast grammar (superceded by ast.Constant) in cpy38 + return node.n + elif isinstance(node, ast.Index): + # ast.Index was removed from ast grammar in cpy39 + return _convert(node.value) + elif isinstance(node, ast.ExtSlice): + # ast.ExtSlice was removed from ast grammar in cpy39 + return tuple(map(_convert, node.dims)) return _convert_signed_num(node) return _convert(node_or_string) From 996cbf53c3103d63cb2a20e8ae3b339ff654a0cb Mon Sep 17 00:00:00 2001 From: telamonian Date: Tue, 13 Oct 2020 00:54:11 -0400 Subject: [PATCH 3/9] added unittests with full coverage for literal_eval_index --- ndindex/literal_eval_index.py | 17 +++--- ndindex/tests/test_literal_eval_index.py | 66 ++++++++++++++++++++++++ setup.py | 6 ++- 3 files changed, 79 insertions(+), 10 deletions(-) create mode 100644 ndindex/tests/test_literal_eval_index.py diff --git a/ndindex/literal_eval_index.py b/ndindex/literal_eval_index.py index e9f274e7..9316e99b 100644 --- a/ndindex/literal_eval_index.py +++ b/ndindex/literal_eval_index.py @@ -1,6 +1,6 @@ import ast -def literal_eval_slice(node_or_string): +def literal_eval_index(node_or_string): """ "Safely" (needs validation) evaluate an expression node or a string containing a (limited subset) of valid numpy index or slice expressions. @@ -21,8 +21,9 @@ def _convert_num(node): if isinstance(node.value, (int, float, complex)): return node.value elif isinstance(node, ast.Num): + # ast.Num was removed from ast grammar in cpy38 return node.n - raise ValueError('malformed node or string: ' + repr(node)) + _raise_malformed_node(node) def _convert_signed_num(node): if isinstance(node, ast.UnaryOp) and isinstance(node.op, (ast.UAdd, ast.USub)): operand = _convert_num(node.operand) @@ -33,9 +34,7 @@ def _convert_signed_num(node): return _convert_num(node) def _convert(node): - if isinstance(node, ast.Constant): - return node.value - elif isinstance(node, ast.Tuple): + if isinstance(node, ast.Tuple): return tuple(map(_convert, node.elts)) elif isinstance(node, ast.List): return list(map(_convert, node.elts)) @@ -45,9 +44,11 @@ def _convert(node): _convert(node.upper) if node.upper is not None else None, _convert(node.step) if node.step is not None else None, ) - elif isinstance(node, ast.Num): - # ast.Num was removed from ast grammar (superceded by ast.Constant) in cpy38 - return node.n + elif isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id == 'slice' and node.keywords == []: + # support for parsing slices written out as 'slice(...)' objects + return slice(*map(_convert, node.args)) + elif isinstance(node, ast.NameConstant) and node.value is None: + return None elif isinstance(node, ast.Index): # ast.Index was removed from ast grammar in cpy39 return _convert(node.value) diff --git a/ndindex/tests/test_literal_eval_index.py b/ndindex/tests/test_literal_eval_index.py new file mode 100644 index 00000000..31ff2942 --- /dev/null +++ b/ndindex/tests/test_literal_eval_index.py @@ -0,0 +1,66 @@ +import ast +from hypothesis import given, example +from hypothesis.strategies import one_of +import pytest + +from ..literal_eval_index import literal_eval_index +from .helpers import ints, slices, tuples, _doesnt_raise + +Tuples = tuples(one_of( + # no ellipses support for now + # ellipses(), + ints(), + slices(), +)).filter(_doesnt_raise) + +ndindexStrs = one_of( + # ellipses(), + ints(), + slices(), + Tuples, +).map(lambda x: f'{x}') + +class _Dummy: + def __getitem__(self, x): + return x +_dummy = _Dummy() + +@example('3') +@example('-3') +@example('+3') +@example('3:4') +@example('3:-4') +@example('3, 5, 14, 1') +@example('3, -5, 14, -1') +@example('3:15, 5, 14:99, 1') +@example('3:15, -5, 14:-99, 1') +@example(':15, -5, 14:-99:3, 1') +@example('3:15, -5, :, [1,2,3]') +@example('slice(None)') +@example('slice(None, None)') +@example('slice(None, None, None)') +@example('slice(14)') +@example('slice(12, 14)') +@example('slice(12, 72, 14)') +@example('slice(-12, -72, 14)') +@example('3:15, -5, slice(12, -14), (1,2,3)') +@given(ndindexStrs) +def test_literal_eval_index_hypothesis(ixStr): + assert eval(f'_dummy[{ixStr}]') == literal_eval_index(ixStr) + +def test_literal_eval_index_malformed_raise(): + with pytest.raises(ValueError): + # we don't allow the bitwise not unary op + ixStr = '~3' + literal_eval_index(ixStr) + +def test_literal_eval_index_ensure_coverage(): + # ensure full coverage, regarless of cpy version and accompanying changes to the ast grammar + for node in ( + ast.Constant(7), + ast.Num(7), + ast.Index(ast.Constant(7)), + ): + assert literal_eval_index(node) == 7 + + assert literal_eval_index(ast.ExtSlice((ast.Constant(7), ast.Constant(7), ast.Constant(7)))) == (7, 7, 7) diff --git a/setup.py b/setup.py index 1b6035b6..f774bf11 100644 --- a/setup.py +++ b/setup.py @@ -20,8 +20,10 @@ "sympy", ], tests_require=[ - 'pytest', - 'hypothesis', + "pytest", + "pytest-flakes", + "pytest-tornasync", + "hypothesis", ], classifiers=[ "Programming Language :: Python :: 3", From 0f30a09a73a3f7b63575db5f5e5f3dcc3cd545ec Mon Sep 17 00:00:00 2001 From: telamonian Date: Tue, 13 Oct 2020 01:26:41 -0400 Subject: [PATCH 4/9] added support for .. and Ellipsis to literal_eval_index --- ndindex/literal_eval_index.py | 7 +++++++ ndindex/tests/test_literal_eval_index.py | 11 +++++++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/ndindex/literal_eval_index.py b/ndindex/literal_eval_index.py index 9316e99b..dfeb30b1 100644 --- a/ndindex/literal_eval_index.py +++ b/ndindex/literal_eval_index.py @@ -48,7 +48,14 @@ def _convert(node): # support for parsing slices written out as 'slice(...)' objects return slice(*map(_convert, node.args)) elif isinstance(node, ast.NameConstant) and node.value is None: + # support for literal None in slices, eg 'slice(None, ...)' return None + elif isinstance(node, ast.Ellipsis): + # support for three dot '...' ellipsis syntax + return ... + elif isinstance(node, ast.Name) and node.id == 'Ellipsis': + # support for 'Ellipsis' ellipsis syntax + return ... elif isinstance(node, ast.Index): # ast.Index was removed from ast grammar in cpy39 return _convert(node.value) diff --git a/ndindex/tests/test_literal_eval_index.py b/ndindex/tests/test_literal_eval_index.py index 31ff2942..c0464e7a 100644 --- a/ndindex/tests/test_literal_eval_index.py +++ b/ndindex/tests/test_literal_eval_index.py @@ -4,17 +4,16 @@ import pytest from ..literal_eval_index import literal_eval_index -from .helpers import ints, slices, tuples, _doesnt_raise +from .helpers import ellipses, ints, slices, tuples, _doesnt_raise Tuples = tuples(one_of( - # no ellipses support for now - # ellipses(), + ellipses(), ints(), slices(), )).filter(_doesnt_raise) ndindexStrs = one_of( - # ellipses(), + ellipses(), ints(), slices(), Tuples, @@ -27,6 +26,8 @@ def __getitem__(self, x): @example('3') @example('-3') +@example('...') +@example('Ellipsis') @example('+3') @example('3:4') @example('3:-4') @@ -44,6 +45,8 @@ def __getitem__(self, x): @example('slice(12, 72, 14)') @example('slice(-12, -72, 14)') @example('3:15, -5, slice(12, -14), (1,2,3)') +@example('..., -5, slice(12, -14), (1,2,3)') +@example('3:15, -5, slice(12, -14), (1,2,3), Ellipsis') @given(ndindexStrs) def test_literal_eval_index_hypothesis(ixStr): assert eval(f'_dummy[{ixStr}]') == literal_eval_index(ixStr) From 447cda0f7410d9473791df51105d5cb75923e752 Mon Sep 17 00:00:00 2001 From: telamonian Date: Tue, 13 Oct 2020 18:54:21 -0400 Subject: [PATCH 5/9] first attempt at disallowing nested tuple indices in literal_eval_index - see https://github.com/Quansight/ndindex/pull/91#discussion_r504226868 --- ndindex/literal_eval_index.py | 15 +++++++++++++++ ndindex/tests/test_literal_eval_index.py | 5 ++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/ndindex/literal_eval_index.py b/ndindex/literal_eval_index.py index dfeb30b1..ae2a7e43 100644 --- a/ndindex/literal_eval_index.py +++ b/ndindex/literal_eval_index.py @@ -1,5 +1,16 @@ import ast +class _Guard: + def __init__(self): + self.val = False + + def __call__(self): + if self.val: + return True + else: + self.val = True + return False + def literal_eval_index(node_or_string): """ "Safely" (needs validation) evaluate an expression node or a string containing @@ -33,8 +44,12 @@ def _convert_signed_num(node): return - operand return _convert_num(node) + _nested_tuple_guard = _Guard() def _convert(node): if isinstance(node, ast.Tuple): + if _nested_tuple_guard(): + raise ValueError(f'tuples inside of tuple indices are not supported: {node!r}') + return tuple(map(_convert, node.elts)) elif isinstance(node, ast.List): return list(map(_convert, node.elts)) diff --git a/ndindex/tests/test_literal_eval_index.py b/ndindex/tests/test_literal_eval_index.py index c0464e7a..7e4d46a0 100644 --- a/ndindex/tests/test_literal_eval_index.py +++ b/ndindex/tests/test_literal_eval_index.py @@ -44,9 +44,8 @@ def __getitem__(self, x): @example('slice(12, 14)') @example('slice(12, 72, 14)') @example('slice(-12, -72, 14)') -@example('3:15, -5, slice(12, -14), (1,2,3)') -@example('..., -5, slice(12, -14), (1,2,3)') -@example('3:15, -5, slice(12, -14), (1,2,3), Ellipsis') +@example('3:15, -5, slice(-12, -72, 14), Ellipsis') +@example('..., 3:15, -5, slice(-12, -72, 14)') @given(ndindexStrs) def test_literal_eval_index_hypothesis(ixStr): assert eval(f'_dummy[{ixStr}]') == literal_eval_index(ixStr) From 0b6e7c8860914f72c64122e697be2b76c28b50c3 Mon Sep 17 00:00:00 2001 From: telamonian Date: Tue, 13 Oct 2020 19:48:11 -0400 Subject: [PATCH 6/9] renamed `eval_literal_index` -> `parse_index`, moved it to `ndindex.py` --- ndindex/literal_eval_index.py | 82 ---------------------- ndindex/ndindex.py | 86 ++++++++++++++++++++++++ ndindex/tests/test_literal_eval_index.py | 68 ------------------- ndindex/tests/test_ndindex.py | 86 ++++++++++++++++++++++-- 4 files changed, 166 insertions(+), 156 deletions(-) delete mode 100644 ndindex/literal_eval_index.py delete mode 100644 ndindex/tests/test_literal_eval_index.py diff --git a/ndindex/literal_eval_index.py b/ndindex/literal_eval_index.py deleted file mode 100644 index ae2a7e43..00000000 --- a/ndindex/literal_eval_index.py +++ /dev/null @@ -1,82 +0,0 @@ -import ast - -class _Guard: - def __init__(self): - self.val = False - - def __call__(self): - if self.val: - return True - else: - self.val = True - return False - -def literal_eval_index(node_or_string): - """ - "Safely" (needs validation) evaluate an expression node or a string containing - a (limited subset) of valid numpy index or slice expressions. - """ - if isinstance(node_or_string, str): - node_or_string = ast.parse('dummy[{}]'.format(node_or_string.lstrip(" \t")) , mode='eval') - if isinstance(node_or_string, ast.Expression): - node_or_string = node_or_string.body - if isinstance(node_or_string, ast.Subscript): - node_or_string = node_or_string.slice - - def _raise_malformed_node(node): - raise ValueError(f'malformed node or string: {node!r}') - - # from cpy37, should work until they remove ast.Num (not until cpy310) - def _convert_num(node): - if isinstance(node, ast.Constant): - if isinstance(node.value, (int, float, complex)): - return node.value - elif isinstance(node, ast.Num): - # ast.Num was removed from ast grammar in cpy38 - return node.n - _raise_malformed_node(node) - def _convert_signed_num(node): - if isinstance(node, ast.UnaryOp) and isinstance(node.op, (ast.UAdd, ast.USub)): - operand = _convert_num(node.operand) - if isinstance(node.op, ast.UAdd): - return + operand - else: - return - operand - return _convert_num(node) - - _nested_tuple_guard = _Guard() - def _convert(node): - if isinstance(node, ast.Tuple): - if _nested_tuple_guard(): - raise ValueError(f'tuples inside of tuple indices are not supported: {node!r}') - - return tuple(map(_convert, node.elts)) - elif isinstance(node, ast.List): - return list(map(_convert, node.elts)) - elif isinstance(node, ast.Slice): - return slice( - _convert(node.lower) if node.lower is not None else None, - _convert(node.upper) if node.upper is not None else None, - _convert(node.step) if node.step is not None else None, - ) - elif isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id == 'slice' and node.keywords == []: - # support for parsing slices written out as 'slice(...)' objects - return slice(*map(_convert, node.args)) - elif isinstance(node, ast.NameConstant) and node.value is None: - # support for literal None in slices, eg 'slice(None, ...)' - return None - elif isinstance(node, ast.Ellipsis): - # support for three dot '...' ellipsis syntax - return ... - elif isinstance(node, ast.Name) and node.id == 'Ellipsis': - # support for 'Ellipsis' ellipsis syntax - return ... - elif isinstance(node, ast.Index): - # ast.Index was removed from ast grammar in cpy39 - return _convert(node.value) - elif isinstance(node, ast.ExtSlice): - # ast.ExtSlice was removed from ast grammar in cpy39 - return tuple(map(_convert, node.dims)) - - return _convert_signed_num(node) - return _convert(node_or_string) diff --git a/ndindex/ndindex.py b/ndindex/ndindex.py index cf7688fb..3fa5991e 100644 --- a/ndindex/ndindex.py +++ b/ndindex/ndindex.py @@ -1,3 +1,4 @@ +import ast import inspect import numbers @@ -65,6 +66,91 @@ def __init__(self, f): def __get__(self, obj, owner): return self.f(owner) +class _Guard: + def __init__(self): + self.val = False + + def __call__(self): + if self.val: + return True + else: + self.val = True + return False + +def parse_index(node_or_string): + """ + "Safely" (needs validation) evaluate an expression node or a string containing + a (limited subset) of valid numpy index or slice expressions. + """ + if isinstance(node_or_string, str): + node_or_string = ast.parse('dummy[{}]'.format(node_or_string.lstrip(" \t")) , mode='eval') + if isinstance(node_or_string, ast.Expression): + node_or_string = node_or_string.body + if isinstance(node_or_string, ast.Subscript): + node_or_string = node_or_string.slice + + def _raise_malformed_node(node): + raise ValueError(f'malformed node or string: {node!r}') + def _raise_nested_tuple_node(node): + raise ValueError(f'tuples inside of tuple indices are not supported: {node!r}') + + # from cpy37, should work until they remove ast.Num (not until cpy310) + def _convert_num(node): + if isinstance(node, ast.Constant): + if isinstance(node.value, (int, float, complex)): + return node.value + elif isinstance(node, ast.Num): + # ast.Num was removed from ast grammar in cpy38 + return node.n + _raise_malformed_node(node) + def _convert_signed_num(node): + if isinstance(node, ast.UnaryOp) and isinstance(node.op, (ast.UAdd, ast.USub)): + operand = _convert_num(node.operand) + if isinstance(node.op, ast.UAdd): + return + operand + else: + return - operand + return _convert_num(node) + + _nested_tuple_guard = _Guard() + def _convert(node): + if isinstance(node, ast.Tuple): + if _nested_tuple_guard(): + _raise_nested_tuple_node(node) + + return tuple(map(_convert, node.elts)) + elif isinstance(node, ast.List): + return list(map(_convert, node.elts)) + elif isinstance(node, ast.Slice): + return slice( + _convert(node.lower) if node.lower is not None else None, + _convert(node.upper) if node.upper is not None else None, + _convert(node.step) if node.step is not None else None, + ) + elif isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id == 'slice' and node.keywords == []: + # support for parsing slices written out as 'slice(...)' objects + return slice(*map(_convert, node.args)) + elif isinstance(node, ast.NameConstant) and node.value is None: + # support for literal None in slices, eg 'slice(None, ...)' + return None + elif isinstance(node, ast.Ellipsis): + # support for three dot '...' ellipsis syntax + return ... + elif isinstance(node, ast.Name) and node.id == 'Ellipsis': + # support for 'Ellipsis' ellipsis syntax + return ... + elif isinstance(node, ast.Index): + # ast.Index was removed from ast grammar in cpy39 + return _convert(node.value) + elif isinstance(node, ast.ExtSlice): + # ast.ExtSlice was removed from ast grammar in cpy39 + _nested_tuple_guard() + return tuple(map(_convert, node.dims)) + + return _convert_signed_num(node) + return _convert(node_or_string) + + class NDIndex: """ Represents an index into an nd-array (i.e., a numpy array). diff --git a/ndindex/tests/test_literal_eval_index.py b/ndindex/tests/test_literal_eval_index.py deleted file mode 100644 index 7e4d46a0..00000000 --- a/ndindex/tests/test_literal_eval_index.py +++ /dev/null @@ -1,68 +0,0 @@ -import ast -from hypothesis import given, example -from hypothesis.strategies import one_of -import pytest - -from ..literal_eval_index import literal_eval_index -from .helpers import ellipses, ints, slices, tuples, _doesnt_raise - -Tuples = tuples(one_of( - ellipses(), - ints(), - slices(), -)).filter(_doesnt_raise) - -ndindexStrs = one_of( - ellipses(), - ints(), - slices(), - Tuples, -).map(lambda x: f'{x}') - -class _Dummy: - def __getitem__(self, x): - return x -_dummy = _Dummy() - -@example('3') -@example('-3') -@example('...') -@example('Ellipsis') -@example('+3') -@example('3:4') -@example('3:-4') -@example('3, 5, 14, 1') -@example('3, -5, 14, -1') -@example('3:15, 5, 14:99, 1') -@example('3:15, -5, 14:-99, 1') -@example(':15, -5, 14:-99:3, 1') -@example('3:15, -5, :, [1,2,3]') -@example('slice(None)') -@example('slice(None, None)') -@example('slice(None, None, None)') -@example('slice(14)') -@example('slice(12, 14)') -@example('slice(12, 72, 14)') -@example('slice(-12, -72, 14)') -@example('3:15, -5, slice(-12, -72, 14), Ellipsis') -@example('..., 3:15, -5, slice(-12, -72, 14)') -@given(ndindexStrs) -def test_literal_eval_index_hypothesis(ixStr): - assert eval(f'_dummy[{ixStr}]') == literal_eval_index(ixStr) - -def test_literal_eval_index_malformed_raise(): - with pytest.raises(ValueError): - # we don't allow the bitwise not unary op - ixStr = '~3' - literal_eval_index(ixStr) - -def test_literal_eval_index_ensure_coverage(): - # ensure full coverage, regarless of cpy version and accompanying changes to the ast grammar - for node in ( - ast.Constant(7), - ast.Num(7), - ast.Index(ast.Constant(7)), - ): - assert literal_eval_index(node) == 7 - - assert literal_eval_index(ast.ExtSlice((ast.Constant(7), ast.Constant(7), ast.Constant(7)))) == (7, 7, 7) diff --git a/ndindex/tests/test_ndindex.py b/ndindex/tests/test_ndindex.py index a8ea8ed7..a9c5671b 100644 --- a/ndindex/tests/test_ndindex.py +++ b/ndindex/tests/test_ndindex.py @@ -1,17 +1,34 @@ +import ast import inspect - import numpy as np - -from hypothesis import given, example, settings - +from hypothesis import example, given, settings +from hypothesis.strategies import one_of from pytest import raises, warns -from ..ndindex import ndindex, asshape +from ..ndindex import ndindex, parse_index, asshape from ..integer import Integer from ..ellipsis import ellipsis from ..integerarray import IntegerArray from ..tuple import Tuple -from .helpers import ndindices, check_same, assert_equal +from .helpers import ndindices, check_same, assert_equal, ellipses, ints, slices, tuples, _doesnt_raise + +Tuples = tuples(one_of( + ellipses(), + ints(), + slices(), +)).filter(_doesnt_raise) + +ndindexStrs = one_of( + ellipses(), + ints(), + slices(), + Tuples, +).map(lambda x: f'{x}') + +class _Dummy: + def __getitem__(self, x): + return x +_dummy = _Dummy() @given(ndindices) def test_eq(idx): @@ -77,6 +94,63 @@ def test_ndindex_invalid(): def test_ndindex_ellipsis(): raises(IndexError, lambda: ndindex(ellipsis)) + +@example('3') +@example('-3') +@example('...') +@example('Ellipsis') +@example('+3') +@example('3:4') +@example('3:-4') +@example('3, 5, 14, 1') +@example('3, -5, 14, -1') +@example('3:15, 5, 14:99, 1') +@example('3:15, -5, 14:-99, 1') +@example(':15, -5, 14:-99:3, 1') +@example('3:15, -5, :, [1,2,3]') +@example('slice(None)') +@example('slice(None, None)') +@example('slice(None, None, None)') +@example('slice(14)') +@example('slice(12, 14)') +@example('slice(12, 72, 14)') +@example('slice(-12, -72, 14)') +@example('3:15, -5, slice(-12, -72, 14), Ellipsis') +@example('..., 3:15, -5, slice(-12, -72, 14)') +@given(ndindexStrs) +def test_parse_index_hypothesis(ixStr): + assert eval(f'_dummy[{ixStr}]') == parse_index(ixStr) + +def test_parse_index_malformed_raise(): + # we don't allow the bitwise not unary op + with raises(ValueError): + ixStr = '~3' + parse_index(ixStr) + +def test_parse_index_nested_tuple_raise(): + # we don't allow tuples within tuple indices + with raises(ValueError): + # this will parse as either ast.Index or ast.Slice (depending on cpy version) containing an ast.Tuple + ixStr = '..., -5, slice(12, -14), (1,2,3)' + parse_index(ixStr) + + with raises(ValueError): + # in cpy37, this will parse as ast.ExtSlice containing an ast.Tuple + ixStr = '3:15, -5, :, (1,2,3)' + parse_index(ixStr) + +def test_parse_index_ensure_coverage(): + # ensure full coverage, regarless of cpy version and accompanying changes to the ast grammar + for node in ( + ast.Constant(7), + ast.Num(7), + ast.Index(ast.Constant(7)), + ): + assert parse_index(node) == 7 + + assert parse_index(ast.ExtSlice((ast.Constant(7), ast.Constant(7), ast.Constant(7)))) == (7, 7, 7) + + def test_signature(): sig = inspect.signature(Integer) assert sig.parameters.keys() == {'idx'} From 3733175069b3640e89c5f25c046b588c597114cb Mon Sep 17 00:00:00 2001 From: telamonian Date: Tue, 13 Oct 2020 22:53:52 -0400 Subject: [PATCH 7/9] parse_index now returns an NDIndex --- ndindex/ndindex.py | 2 +- ndindex/tests/helpers.py | 4 +++- ndindex/tests/test_ndindex.py | 2 +- setup.py | 1 + 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/ndindex/ndindex.py b/ndindex/ndindex.py index 3fa5991e..b8f3ec43 100644 --- a/ndindex/ndindex.py +++ b/ndindex/ndindex.py @@ -148,7 +148,7 @@ def _convert(node): return tuple(map(_convert, node.dims)) return _convert_signed_num(node) - return _convert(node_or_string) + return ndindex(_convert(node_or_string)) class NDIndex: diff --git a/ndindex/tests/helpers.py b/ndindex/tests/helpers.py index 93d336b3..aeda6631 100644 --- a/ndindex/tests/helpers.py +++ b/ndindex/tests/helpers.py @@ -24,12 +24,14 @@ def prod(seq): return reduce(mul, seq, 1) +positive_ints = integers(1, 10) nonnegative_ints = integers(0, 10) negative_ints = integers(-10, -1) ints = lambda: one_of(negative_ints, nonnegative_ints) +ints_nonzero = lambda: one_of(negative_ints, positive_ints) def slices(start=one_of(none(), ints()), stop=one_of(none(), ints()), - step=one_of(none(), ints())): + step=one_of(none(), ints_nonzero())): return builds(slice, start, stop, step) ellipses = lambda: just(...) diff --git a/ndindex/tests/test_ndindex.py b/ndindex/tests/test_ndindex.py index a9c5671b..4cd05e33 100644 --- a/ndindex/tests/test_ndindex.py +++ b/ndindex/tests/test_ndindex.py @@ -107,7 +107,7 @@ def test_ndindex_ellipsis(): @example('3:15, 5, 14:99, 1') @example('3:15, -5, 14:-99, 1') @example(':15, -5, 14:-99:3, 1') -@example('3:15, -5, :, [1,2,3]') +@example('3:15, -5, [1,2,3], :') @example('slice(None)') @example('slice(None, None)') @example('slice(None, None, None)') diff --git a/setup.py b/setup.py index f774bf11..135cf003 100644 --- a/setup.py +++ b/setup.py @@ -21,6 +21,7 @@ ], tests_require=[ "pytest", + "pytest-cov", "pytest-flakes", "pytest-tornasync", "hypothesis", From 8345caa9fec40c6d4ff02571c69793f2e56e52f2 Mon Sep 17 00:00:00 2001 From: telamonian Date: Mon, 19 Oct 2020 15:57:12 -0400 Subject: [PATCH 8/9] add coverage pragmas in parse_index for code paths unused in cpy38/39 --- ndindex/ndindex.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ndindex/ndindex.py b/ndindex/ndindex.py index b8f3ec43..feebfeec 100644 --- a/ndindex/ndindex.py +++ b/ndindex/ndindex.py @@ -90,9 +90,9 @@ def parse_index(node_or_string): node_or_string = node_or_string.slice def _raise_malformed_node(node): - raise ValueError(f'malformed node or string: {node!r}') + raise ValueError(f'malformed node or string: {node!r}, {ast.dump(node)!r}') def _raise_nested_tuple_node(node): - raise ValueError(f'tuples inside of tuple indices are not supported: {node!r}') + raise ValueError(f'tuples inside of tuple indices are not supported: {node!r}, {ast.dump(node)!r}') # from cpy37, should work until they remove ast.Num (not until cpy310) def _convert_num(node): @@ -101,7 +101,7 @@ def _convert_num(node): return node.value elif isinstance(node, ast.Num): # ast.Num was removed from ast grammar in cpy38 - return node.n + return node.n # pragma: no cover _raise_malformed_node(node) def _convert_signed_num(node): if isinstance(node, ast.UnaryOp) and isinstance(node.op, (ast.UAdd, ast.USub)): @@ -141,11 +141,11 @@ def _convert(node): return ... elif isinstance(node, ast.Index): # ast.Index was removed from ast grammar in cpy39 - return _convert(node.value) + return _convert(node.value) # pragma: no cover elif isinstance(node, ast.ExtSlice): # ast.ExtSlice was removed from ast grammar in cpy39 - _nested_tuple_guard() - return tuple(map(_convert, node.dims)) + _nested_tuple_guard() # pragma: no cover + return tuple(map(_convert, node.dims)) # pragma: no cover return _convert_signed_num(node) return ndindex(_convert(node_or_string)) From 6174a67889129954e7e125a79b036e4aaedd1c4b Mon Sep 17 00:00:00 2001 From: telamonian Date: Thu, 22 Oct 2020 04:01:09 -0400 Subject: [PATCH 9/9] export to top-level __init__.py --- ndindex/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ndindex/__init__.py b/ndindex/__init__.py index 37667c1b..d17a9f85 100644 --- a/ndindex/__init__.py +++ b/ndindex/__init__.py @@ -1,8 +1,8 @@ __all__ = [] -from .ndindex import ndindex +from .ndindex import parse_index, ndindex -__all__ += ['ndindex'] +__all__ += ['parse_index', 'ndindex'] from .slice import Slice