From 990e7ab81abd2681ecba4f8d1670fbcf11f155ac Mon Sep 17 00:00:00 2001 From: Carlos Cordoba Date: Sat, 4 Feb 2017 14:44:03 -0500 Subject: [PATCH 1/6] Introspection: Remove docstrings.py because we can use a simpler file to patch Jedi --- spyder/utils/introspection/docstrings.py | 285 ----------------------- 1 file changed, 285 deletions(-) delete mode 100644 spyder/utils/introspection/docstrings.py diff --git a/spyder/utils/introspection/docstrings.py b/spyder/utils/introspection/docstrings.py deleted file mode 100644 index e107660e94a..00000000000 --- a/spyder/utils/introspection/docstrings.py +++ /dev/null @@ -1,285 +0,0 @@ -""" -Docstrings are another source of information for functions and classes. -:mod:`jedi.evaluate.dynamic` tries to find all executions of functions, while -the docstring parsing is much easier. There are two different types of -docstrings that |jedi| understands: - -- `Sphinx `_ -- `Epydoc `_ - -For example, the sphinx annotation ``:type foo: str`` clearly states that the -type of ``foo`` is ``str``. - -As an addition to parameter searching, this module also provides return -annotations. -""" - -from ast import literal_eval -import re -from itertools import chain -from textwrap import dedent -from jedi import debug -from jedi.evaluate.cache import memoize_default -from jedi.parser import Parser, load_grammar -from jedi.parser.tree import Class -from jedi.common import indent_block -from jedi.evaluate.iterable import Array, FakeSequence, AlreadyEvaluated - - -DOCSTRING_PARAM_PATTERNS = [ - r'\s*:type\s+%s:\s*([^\n]+)', # Sphinx - r'\s*:param\s+(\w+)\s+%s:[^\n]+', # Sphinx param with type - r'\s*@type\s+%s:\s*([^\n]+)', # Epydoc -] - -DOCSTRING_RETURN_PATTERNS = [ - re.compile(r'\s*:rtype:\s*([^\n]+)', re.M), # Sphinx - re.compile(r'\s*@rtype:\s*([^\n]+)', re.M), # Epydoc -] - -REST_ROLE_PATTERN = re.compile(r':[^`]+:`([^`]+)`') - - -try: - from numpydoc.docscrape import NumpyDocString -except ImportError: - def _search_param_in_numpydocstr(docstr, param_str): - return [] - - def _search_return_in_numpydocstr(docstr): - return [] -else: - def _search_param_in_numpydocstr(docstr, param_str): - """Search `docstr` (in numpydoc format) for type(-s) of `param_str`.""" - params = NumpyDocString(docstr)._parsed_data['Parameters'] - for p_name, p_type, p_descr in params: - if p_name == param_str: - m = re.match('([^,]+(,[^,]+)*?)(,[ ]*optional)?$', p_type) - if m: - p_type = m.group(1) - return _expand_typestr(p_type) - return [] - - def _search_return_in_numpydocstr(docstr): - r""" - Search `docstr` (in numpydoc format) for type(-s) of `param_str`. - """ - doc = NumpyDocString(docstr) - returns = doc._parsed_data['Returns'] - returns += doc._parsed_data['Yields'] - found = [] - for p_name, p_type, p_descr in returns: - if not p_type: - p_type = p_name - p_name = '' - - m = re.match('([^,]+(,[^,]+)*?)$', p_type) - if m: - p_type = m.group(1) - found.extend(_expand_typestr(p_type)) - return found - - -def _expand_typestr(p_type): - """ - Attempts to interpret the possible types - """ - # Check if alternative types are specified - if re.search('\\bor\\b', p_type): - types = [t.strip() for t in p_type.split('or')] - # Check if type has a set of valid literal values - elif p_type.startswith('{'): - # python2 does not support literal set evals - # workaround this by using lists instead - p_type = p_type.replace('{', '[').replace('}', ']') - types = set(type(x).__name__ for x in literal_eval(p_type)) - types = list(types) - # Otherwise just return the typestr wrapped in a list - else: - types = [p_type] - return types - - -def _search_param_in_docstr(docstr, param_str): - """ - Search `docstr` for type(-s) of `param_str`. - - >>> _search_param_in_docstr(':type param: int', 'param') - ['int'] - >>> _search_param_in_docstr('@type param: int', 'param') - ['int'] - >>> _search_param_in_docstr( - ... ':type param: :class:`threading.Thread`', 'param') - ['threading.Thread'] - >>> bool(_search_param_in_docstr('no document', 'param')) - False - >>> _search_param_in_docstr(':param int param: some description', 'param') - ['int'] - """ - # look at #40 to see definitions of those params - - # Check for Sphinx/Epydoc params - patterns = [re.compile(p % re.escape(param_str)) - for p in DOCSTRING_PARAM_PATTERNS] - - found = None - for pattern in patterns: - match = pattern.search(docstr) - if match: - found = [_strip_rst_role(match.group(1))] - break - if found is not None: - return found - - # Check for numpy style params - found = _search_param_in_numpydocstr(docstr, param_str) - if found is not None: - return found - - return [] - - -def _strip_rst_role(type_str): - """ - Strip off the part looks like a ReST role in `type_str`. - - >>> _strip_rst_role(':class:`ClassName`') # strip off :class: - 'ClassName' - >>> _strip_rst_role(':py:obj:`module.Object`') # works with domain - 'module.Object' - >>> _strip_rst_role('ClassName') # do nothing when not ReST role - 'ClassName' - - See also: - http://sphinx-doc.org/domains.html#cross-referencing-python-objects - """ - match = REST_ROLE_PATTERN.match(type_str) - if match: - return match.group(1) - else: - return type_str - - -def _evaluate_for_statement_string(evaluator, string, module): - if string is None: - return [] - - code = dedent(""" - def pseudo_docstring_stuff(): - # Create a pseudo function for docstring statements. - %s - """) - - for element in re.findall('((?:\w+\.)*\w+)\.', string): - # Try to import module part in dotted name. - # (e.g., 'threading' in 'threading.Thread'). - string = 'import %s\n' % element + string - - # Take the default grammar here, if we load the Python 2.7 grammar here, it - # will be impossible to use `...` (Ellipsis) as a token. Docstring types - # don't need to conform with the current grammar. - p = Parser(load_grammar(), code % indent_block(string)) - try: - pseudo_cls = p.module.subscopes[0] - # First pick suite, then simple_stmt (-2 for DEDENT) and then the node, - # which is also not the last item, because there's a newline. - stmt = pseudo_cls.children[-1].children[-2].children[-2] - except (AttributeError, IndexError): - type_list = [] - else: - # Use the module of the param. - # TODO this module is not the module of the param in case of a function - # call. In that case it's the module of the function call. - # stuffed with content from a function call. - pseudo_cls.parent = module - type_list = _execute_types_in_stmt(evaluator, stmt) - return type_list - - -def _execute_types_in_stmt(evaluator, stmt): - """ - Executing all types or general elements that we find in a statement. This - doesn't include tuple, list and dict literals, because the stuff they - contain is executed. (Used as type information). - """ - definitions = evaluator.eval_element(stmt) - types_list = [_execute_array_values(evaluator, d) for d in definitions] - type_list = list(chain.from_iterable(types_list)) - return type_list - - -def _execute_array_values(evaluator, array): - """ - Tuples indicate that there's not just one return value, but the listed - ones. `(str, int)` means that it returns a tuple with both types. - """ - if isinstance(array, Array): - values = [] - for types in array.py__iter__(): - objects = set(chain.from_iterable(_execute_array_values(evaluator, typ) for typ in types)) - values.append(AlreadyEvaluated(objects)) - return [FakeSequence(evaluator, values, array.type)] - else: - return evaluator.execute(array) - - -@memoize_default(None, evaluator_is_first_arg=True) -def follow_param(evaluator, param): - """ - Determines a set of potential types for `param` using docstring hints - - :type evaluator: jedi.evaluate.Evaluator - :type param: jedi.parser.tree.Param - - :rtype: list - """ - def eval_docstring(docstr): - param_str = str(param.name) - return set( - [p for string in _search_param_in_docstr(docstr, param_str) - for p in _evaluate_for_statement_string(evaluator, string, module)] - ) - func = param.parent_function - module = param.get_parent_until() - - docstr = func.raw_doc - types = eval_docstring(docstr) - if func.name.value == '__init__': - cls = func.get_parent_until(Class) - if cls.type == 'classdef': - types |= eval_docstring(cls.raw_doc) - - return types - - -@memoize_default(None, evaluator_is_first_arg=True) -def find_return_types(evaluator, func): - """ - Determines a set of potential return types for `func` using docstring hints - - :type evaluator: jedi.evaluate.Evaluator - :type param: jedi.parser.tree.Param - - :rtype: list - """ - def search_return_in_docstr(docstr): - # Check for Sphinx/Epydoc return hint - for p in DOCSTRING_RETURN_PATTERNS: - match = p.search(docstr) - if match: - return [_strip_rst_role(match.group(1))] - found = [] - if not found: - # Check for numpy style return hint - found = _search_return_in_numpydocstr(docstr) - return found - - docstr = func.raw_doc - module = func.get_parent_until() - types = [] - for type_str in search_return_in_docstr(docstr): - type_ = _evaluate_for_statement_string(evaluator, type_str, module) - types.extend(type_) - debug.dbg('DOC!!!!!!!!!!!!!! wow types?: %s in %s',types, func) - return types - From 99c3b1f67c15b474a553f2e4da265feceaea3e17 Mon Sep 17 00:00:00 2001 From: Carlos Cordoba Date: Sat, 4 Feb 2017 14:46:34 -0500 Subject: [PATCH 2/6] Introspection: Add numpy_docstr.py with the necessary functions to patch Jedi for Numpy and Matplotlib --- spyder/utils/introspection/numpy_docstr.py | 150 +++++++++++++++++++++ 1 file changed, 150 insertions(+) create mode 100644 spyder/utils/introspection/numpy_docstr.py diff --git a/spyder/utils/introspection/numpy_docstr.py b/spyder/utils/introspection/numpy_docstr.py new file mode 100644 index 00000000000..c3ff0e83c1d --- /dev/null +++ b/spyder/utils/introspection/numpy_docstr.py @@ -0,0 +1,150 @@ +# -*- coding: utf-8 -*- +# +# Copyright © Spyder Project Contributors +# Licensed under the terms of the MIT License +# (see spyder/__init__.py for details) + +# Contents in this file are taken from +# +# https://github.com/davidhalter/jedi/pull/796 +# +# to patch Jedi 0.9.0 (it probably doesn't work with +# higher versions) + + +from ast import literal_eval +import re + +from jedi._compatibility import is_py3 +from jedi.evaluate.cache import memoize_default +from jedi.evaluate.docstrings import (_evaluate_for_statement_string, + _strip_rst_role, + DOCSTRING_RETURN_PATTERNS) +from numpydoc.docscrape import NumpyDocString + + +def _expand_typestr(p_type): + """ + Attempts to interpret the possible types + """ + # Check if alternative types are specified + if re.search('\\bor\\b', p_type): + types = [t.strip() for t in p_type.split('or')] + # Check if type has a set of valid literal values + elif p_type.startswith('{'): + if not is_py3: + # python2 does not support literal set evals + # workaround this by using lists instead + p_type = p_type.replace('{', '[').replace('}', ']') + types = set(type(x).__name__ for x in literal_eval(p_type)) + types = list(types) + # Otherwise just return the typestr wrapped in a list + else: + types = [p_type] + return types + + +def _search_param_in_numpydocstr(docstr, param_str): + r""" + Search `docstr` (in numpydoc format) for type(-s) of `param_str`. + >>> from jedi.evaluate.docstrings import * # NOQA + >>> from jedi.evaluate.docstrings import _search_param_in_numpydocstr + >>> docstr = ( + ... 'Parameters\n' + ... '----------\n' + ... 'x : ndarray\n' + ... 'y : int or str or list\n' + ... 'z : {"foo", "bar", 100500}, optional\n' + ... ) + >>> _search_param_in_numpydocstr(docstr, 'x') + ['ndarray'] + >>> sorted(_search_param_in_numpydocstr(docstr, 'y')) + ['int', 'list', 'str'] + >>> sorted(_search_param_in_numpydocstr(docstr, 'z')) + ['int', 'str'] + """ + params = NumpyDocString(docstr)._parsed_data['Parameters'] + for p_name, p_type, p_descr in params: + if p_name == param_str: + m = re.match('([^,]+(,[^,]+)*?)(,[ ]*optional)?$', p_type) + if m: + p_type = m.group(1) + return _expand_typestr(p_type) + return [] + + +def _search_return_in_numpydocstr(docstr): + r""" + Search `docstr` (in numpydoc format) for type(-s) of `param_str`. + >>> from jedi.evaluate.docstrings import * # NOQA + >>> from jedi.evaluate.docstrings import _search_return_in_numpydocstr + >>> from jedi.evaluate.docstrings import _expand_typestr + >>> docstr = ( + ... 'Returns\n' + ... '----------\n' + ... 'int\n' + ... ' can return an anoymous integer\n' + ... 'out : ndarray\n' + ... ' can return a named value\n' + ... ) + >>> _search_return_in_numpydocstr(docstr) + ['int', 'ndarray'] + """ + doc = NumpyDocString(docstr) + returns = doc._parsed_data['Returns'] + returns += doc._parsed_data['Yields'] + found = [] + for p_name, p_type, p_descr in returns: + if not p_type: + p_type = p_name + p_name = '' + + m = re.match('([^,]+(,[^,]+)*?)$', p_type) + if m: + p_type = m.group(1) + found.extend(_expand_typestr(p_type)) + return found + + +@memoize_default(None, evaluator_is_first_arg=True) +def find_return_types(evaluator, func): + """ + Determines a set of potential return types for `func` using docstring hints + :type evaluator: jedi.evaluate.Evaluator + :type param: jedi.parser.tree.Param + :rtype: list + >>> from jedi.evaluate.docstrings import * # NOQA + >>> from jedi.evaluate.docstrings import _search_param_in_docstr + >>> from jedi.evaluate.docstrings import _evaluate_for_statement_string + >>> from jedi.evaluate.docstrings import _search_return_in_gooogledocstr + >>> from jedi.evaluate.docstrings import _search_return_in_numpydocstr + >>> from jedi._compatibility import builtins + >>> source = open(jedi.evaluate.docstrings.__file__.replace('.pyc', '.py'), 'r').read() + >>> script = jedi.Script(source) + >>> evaluator = script._evaluator + >>> func = script._get_module().names_dict['find_return_types'][0].parent + >>> types = find_return_types(evaluator, func) + >>> print('types = %r' % (types,)) + >>> assert len(types) == 1 + >>> assert types[0].base.obj is builtins.list + """ + def search_return_in_docstr(docstr): + # Check for Sphinx/Epydoc return hint + for p in DOCSTRING_RETURN_PATTERNS: + match = p.search(docstr) + if match: + return [_strip_rst_role(match.group(1))] + found = [] + + if not found: + # Check for numpy style return hint + found = _search_return_in_numpydocstr(docstr) + return found + + docstr = func.raw_doc + module = func.get_parent_until() + types = [] + for type_str in search_return_in_docstr(docstr): + type_ = _evaluate_for_statement_string(evaluator, type_str, module) + types.extend(type_) + return types From 4bcda142db81e135c088aee26d504b93f834bff4 Mon Sep 17 00:00:00 2001 From: Carlos Cordoba Date: Sat, 4 Feb 2017 14:49:16 -0500 Subject: [PATCH 3/6] Introspection: Use numpy_docstr in jedi_patch.py --- spyder/utils/introspection/jedi_patch.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/spyder/utils/introspection/jedi_patch.py b/spyder/utils/introspection/jedi_patch.py index 347f96c92d4..1790be10d7c 100644 --- a/spyder/utils/introspection/jedi_patch.py +++ b/spyder/utils/introspection/jedi_patch.py @@ -28,8 +28,11 @@ def apply(): raise ImportError("jedi %s can't be patched" % jedi.__version__) # [1] Adding numpydoc type returns to docstrings - from spyder.utils.introspection import docstrings - jedi.evaluate.representation.docstrings = docstrings + from spyder.utils.introspection import numpy_docstr + jedi.evaluate.representation.docstrings._search_param_in_numpydocstr = \ + numpy_docstr._search_param_in_numpydocstr + jedi.evaluate.representation.docstrings.find_return_types = \ + numpy_docstr.find_return_types # [2] Adding type returns for compiled objects in jedi # Patching jedi.evaluate.compiled.CompiledObject... @@ -42,8 +45,8 @@ def _execute_function(self, evaluator, params): if self.type != 'funcdef': return # patching docstrings here - from spyder.utils.introspection import docstrings - types = docstrings.find_return_types(evaluator, self) + from spyder.utils.introspection import numpy_docstr + types = numpy_docstr.find_return_types(evaluator, self) if types: for result in types: debug.dbg('docstrings type return: %s in %s', result, self) @@ -111,7 +114,7 @@ def calculate_children(evaluator, children): # [4] Fixing introspection for matplotlib Axes objects # Patching jedi.evaluate.precedence... from jedi.evaluate.representation import ( - tree, InstanceName, Instance, compiled, FunctionExecution, InstanceElement) + InstanceName, Instance, compiled, FunctionExecution, InstanceElement) def get_instance_el(evaluator, instance, var, is_class_var=False): """ From cce1b00e39ce9e309a9d3b39adf4ba8a4b3abf31 Mon Sep 17 00:00:00 2001 From: Carlos Cordoba Date: Sat, 4 Feb 2017 14:49:37 -0500 Subject: [PATCH 4/6] Introspection: Improve a test --- spyder/utils/introspection/test/test_jedi_plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spyder/utils/introspection/test/test_jedi_plugin.py b/spyder/utils/introspection/test/test_jedi_plugin.py index 68d8f42a6a2..eb8e870939d 100644 --- a/spyder/utils/introspection/test/test_jedi_plugin.py +++ b/spyder/utils/introspection/test/test_jedi_plugin.py @@ -55,7 +55,7 @@ def test_get_path(): source_code = 'from spyder.utils.introspection.manager import CodeInfo' path, line_nr = p.get_definition(CodeInfo('definition', source_code, len(source_code), __file__)) - assert 'utils.py' in path and 'introspection' in path + assert 'utils' in path and 'introspection' in path def test_get_docstring(): From 43f22281b10a7ad0e89f7dc29327d186818f26ee Mon Sep 17 00:00:00 2001 From: Carlos Cordoba Date: Sat, 4 Feb 2017 15:48:10 -0500 Subject: [PATCH 5/6] Fix Jedi to 0.9.0 because several tests are failing with 0.10.0 --- README.md | 2 +- continuous_integration/conda-recipes/spyder/meta.yaml | 2 +- doc/installation.rst | 2 +- setup.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 2e110bdde86..4011720acdc 100644 --- a/README.md +++ b/README.md @@ -143,7 +143,7 @@ a Python version greater than 2.7 (Python 3.2 is not supported anymore). * **Python** 2.7 or 3.3+ * **PyQt5** 5.2+ or **PyQt4** 4.6+: PyQt5 is recommended. * **qtconsole** 4.2.0+: Enhanced Python interpreter. -* **Rope** and **Jedi**: Editor code completion, calltips +* **Rope** and **Jedi** 0.9.0: Editor code completion, calltips and go-to-definition. * **Pyflakes**: Real-time code analysis. * **Sphinx**: Rich text mode for the Help pane. diff --git a/continuous_integration/conda-recipes/spyder/meta.yaml b/continuous_integration/conda-recipes/spyder/meta.yaml index e0e971dbab8..41f90cbaa8e 100644 --- a/continuous_integration/conda-recipes/spyder/meta.yaml +++ b/continuous_integration/conda-recipes/spyder/meta.yaml @@ -23,7 +23,7 @@ requirements: - rope 0.9.* # [py34 or py35] - rope # [py27] - pyflakes - - jedi + - jedi 0.9.* - qtconsole - nbconvert - pygments diff --git a/doc/installation.rst b/doc/installation.rst index 47abb53ec5b..678ba8f381a 100644 --- a/doc/installation.rst +++ b/doc/installation.rst @@ -161,7 +161,7 @@ The requirements to run Spyder are: enhanced Python interpreter. * `Rope `_ >=0.9.4 and - `Jedi ` 0.8.1 -- for code completion, + `Jedi ` 0.9.0 -- for code completion, go-to-definition and calltips on the Editor. * `Pyflakes `_ -- for real-time diff --git a/setup.py b/setup.py index 0bcbc90bd91..09860cfd506 100644 --- a/setup.py +++ b/setup.py @@ -271,7 +271,7 @@ def run(self): install_requires = [ 'rope_py3k' if PY3 else 'rope>=0.9.4', - 'jedi', + 'jedi==0.9.0', 'pyflakes', 'pygments>=2.0', 'qtconsole>=4.2.0', From e707168cf268369327127a43c56663f28c7ccb87 Mon Sep 17 00:00:00 2001 From: Carlos Cordoba Date: Sat, 4 Feb 2017 15:55:36 -0500 Subject: [PATCH 6/6] Introspection: Change Jedi required version to 0.9.0 --- spyder/utils/introspection/manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spyder/utils/introspection/manager.py b/spyder/utils/introspection/manager.py index edce2dae1c2..eb5df245754 100644 --- a/spyder/utils/introspection/manager.py +++ b/spyder/utils/introspection/manager.py @@ -33,7 +33,7 @@ _("Editor's code completion, go-to-definition and help"), required_version=ROPE_REQVER) -JEDI_REQVER = '>=0.8.1' +JEDI_REQVER = '=0.9.0' dependencies.add('jedi', _("Editor's code completion, go-to-definition and help"), required_version=JEDI_REQVER)