From 6d455c809105faa9f89945f12b4893759965bc40 Mon Sep 17 00:00:00 2001 From: Min RK Date: Wed, 27 Sep 2017 13:11:42 +0200 Subject: [PATCH 01/16] use cloudpickle instead of deprecated serialize to send data to the frontend --- requirements/requirements.txt | 1 + setup.py | 1 + spyder/plugins/tests/test_ipythonconsole.py | 6 +-- spyder/utils/ipython/spyder_kernel.py | 43 ++++++++++--------- .../ipythonconsole/namespacebrowser.py | 20 ++++----- .../variableexplorer/collectionseditor.py | 1 + 6 files changed, 36 insertions(+), 36 deletions(-) diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 0ed40dff9ab..2685be018b4 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -1,3 +1,4 @@ +cloudpickle rope>=0.9.4 jedi>=0.9.0 pyflakes diff --git a/setup.py b/setup.py index 99a8eade2ef..ac719c53f8c 100644 --- a/setup.py +++ b/setup.py @@ -272,6 +272,7 @@ def run(self): import setuptools # analysis:ignore install_requires = [ + 'cloudpickle', 'rope>=0.10.5', 'jedi>=0.9.0', 'pyflakes', diff --git a/spyder/plugins/tests/test_ipythonconsole.py b/spyder/plugins/tests/test_ipythonconsole.py index c1577f55ecc..0e6c54e1818 100644 --- a/spyder/plugins/tests/test_ipythonconsole.py +++ b/spyder/plugins/tests/test_ipythonconsole.py @@ -11,8 +11,8 @@ import tempfile from textwrap import dedent +import cloudpickle from flaky import flaky -from ipykernel.serialize import serialize_object from pygments.token import Name import pytest from qtpy import PYQT4, PYQT5, PYQT_VERSION @@ -323,7 +323,7 @@ def test_unicode_vars(ipyconsole, qtbot): assert shell.get_value('д') == 10 # Change its value and verify - shell.set_value('д', serialize_object(20)) + shell.set_value('д', cloudpickle.dumps(20)) qtbot.wait(1000) assert shell.get_value('д') == 20 @@ -373,7 +373,7 @@ def test_values_dbg(ipyconsole, qtbot): assert shell.get_value('aa') == 10 # Set value - shell.set_value('aa', serialize_object(20)) + shell.set_value('aa', cloudpickle.dumps(20)) qtbot.wait(1000) assert shell.get_value('aa') == 20 diff --git a/spyder/utils/ipython/spyder_kernel.py b/spyder/utils/ipython/spyder_kernel.py index b9bd9e65d07..b85b19018dd 100644 --- a/spyder/utils/ipython/spyder_kernel.py +++ b/spyder/utils/ipython/spyder_kernel.py @@ -15,11 +15,8 @@ import traceback # Third-party imports -from ipykernel.datapub import publish_data +import cloudpickle from ipykernel.ipkernel import IPythonKernel -import ipykernel.pickleutil -from ipykernel.pickleutil import CannedObject -from ipykernel.serialize import deserialize_object # Check if we are running under an external interpreter IS_EXT_INTERPRETER = os.environ.get('EXTERNAL_INTERPRETER', '').lower() == "true" @@ -43,16 +40,6 @@ make_remote_view) -# XXX --- Disable canning for Numpy arrays for now --- -# This allows getting values between a Python 3 frontend -# and a Python 2 kernel, and viceversa, for several types of -# arrays. -# See this link for interesting ideas on how to solve this -# in the future: -# http://stackoverflow.com/q/30698004/438386 -ipykernel.pickleutil.can_map.pop('numpy.ndarray') - - # Excluded variables from the Variable Explorer (i.e. they are not # shown at all there) EXCLUDED_NAMES = ['In', 'Out', 'exit', 'get_ipython', 'quit'] @@ -149,27 +136,41 @@ def get_var_properties(self): else: return repr(None) + def publish_data(self, data): + """publish native data to the spyder frontend + + Parameters + ---------- + + data: dict + Any dict that is serializable by cloudpickle (should be most things). + """ + serialized = cloudpickle.dumps(data) + self.session.send( + self.iopub_socket, + 'spyder_data_msg', + buffers=[serialized], + ) + def get_value(self, name): """Get the value of a variable""" ns = self._get_current_namespace() value = ns[name] try: - publish_data({'__spy_data__': value}) + self.publish_data({'spy_data': value}) except: # * There is no need to inform users about # these errors. # * value = None makes Spyder to ignore # petitions to display a value value = None - publish_data({'__spy_data__': value}) + self.publish_data({'spy_data': value}) self._do_publish_pdb_state = False def set_value(self, name, value): """Set the value of a variable""" ns = self._get_reference_namespace(name) - value = deserialize_object(value)[0] - if isinstance(value, CannedObject): - value = value.get_object() + value = cloudpickle.loads(value) ns[name] = value def remove_value(self, name): @@ -222,7 +223,7 @@ def publish_pdb_state(self): state = dict(namespace_view = self.get_namespace_view(), var_properties = self.get_var_properties(), step = self._pdb_step) - publish_data({'__spy_pdb_state__': state}) + self.publish_data({'spy_pdb_state': state}) self._do_publish_pdb_state = True def pdb_continue(self): @@ -233,7 +234,7 @@ def pdb_continue(self): Fixes issue 2034 """ if self._pdb_obj: - publish_data({'__spy_pdb_continue__': True}) + self.publish_data({'spy_pdb_continue': True}) # --- For the Help plugin def is_defined(self, obj, force_import=False): diff --git a/spyder/widgets/ipythonconsole/namespacebrowser.py b/spyder/widgets/ipythonconsole/namespacebrowser.py index 3e7444a0b5d..977b17ad65d 100644 --- a/spyder/widgets/ipythonconsole/namespacebrowser.py +++ b/spyder/widgets/ipythonconsole/namespacebrowser.py @@ -12,8 +12,7 @@ from qtpy.QtCore import QEventLoop from qtpy.QtWidgets import QMessageBox -from ipykernel.pickleutil import CannedObject -from ipykernel.serialize import deserialize_object +import cloudpickle from qtconsole.rich_jupyter_widget import RichJupyterWidget from spyder.config.base import _ @@ -159,16 +158,15 @@ def save_namespace(self, filename): return self._kernel_reply # ---- Private API (defined by us) ------------------------------ - def _handle_data_message(self, msg): + def _handle_spyder_data_msg(self, msg): """ - Handle raw (serialized) data sent by the kernel + Handle spyder_data messages - We only handle data asked by Spyder, in case people use - publish_data for other purposes. + Content is cloudpickle-serialized in the buffers. """ # Deserialize data try: - data = deserialize_object(msg['buffers'])[0] + data = cloudpickle.loads(bytes(msg['buffers'][0])) except Exception as msg: self._kernel_value = None self._kernel_reply = repr(msg) @@ -176,22 +174,20 @@ def _handle_data_message(self, msg): return # Receive values asked for Spyder - value = data.get('__spy_data__', None) + value = data.get('spy_data', None) if value is not None: - if isinstance(value, CannedObject): - value = value.get_object() self._kernel_value = value self.sig_got_reply.emit() return # Receive Pdb state and dispatch it - pdb_state = data.get('__spy_pdb_state__', None) + pdb_state = data.get('spy_pdb_state', None) if pdb_state is not None and isinstance(pdb_state, dict): self.refresh_from_pdb(pdb_state) # Run Pdb continue to get to the first breakpoint # Fixes 2034 - pdb_continue = data.get('__spy_pdb_continue__', None) + pdb_continue = data.get('spy_pdb_continue', None) if pdb_continue: self.write_to_stdin('continue') diff --git a/spyder/widgets/variableexplorer/collectionseditor.py b/spyder/widgets/variableexplorer/collectionseditor.py index 4e438dae3ea..5bf63cad590 100644 --- a/spyder/widgets/variableexplorer/collectionseditor.py +++ b/spyder/widgets/variableexplorer/collectionseditor.py @@ -24,6 +24,7 @@ import warnings # Third party imports +import cloudpickle from qtpy.compat import getsavefilename, to_qvariant from qtpy.QtCore import (QAbstractTableModel, QDateTime, QModelIndex, Qt, Signal, Slot) From 280b522325ccd300925ab9401888452b57a45ba3 Mon Sep 17 00:00:00 2001 From: Min RK Date: Wed, 27 Sep 2017 13:28:04 +0200 Subject: [PATCH 02/16] use custom spyder_msg and spyder_msg_type for easier dispatching without sharing a namespace with user messages --- spyder/utils/ipython/spyder_kernel.py | 35 ++++++++----- .../ipythonconsole/namespacebrowser.py | 50 ++++++++----------- 2 files changed, 44 insertions(+), 41 deletions(-) diff --git a/spyder/utils/ipython/spyder_kernel.py b/spyder/utils/ipython/spyder_kernel.py index b85b19018dd..904ee6400a4 100644 --- a/spyder/utils/ipython/spyder_kernel.py +++ b/spyder/utils/ipython/spyder_kernel.py @@ -136,20 +136,30 @@ def get_var_properties(self): else: return repr(None) - def publish_data(self, data): - """publish native data to the spyder frontend + def send_spyder_msg(self, spyder_msg_type, content=None, data=None): + """publish custom messages to the spyder frontend Parameters ---------- - data: dict - Any dict that is serializable by cloudpickle (should be most things). + spyder_msg_type: str + The spyder message type + content: dict + The (JSONable) content of the message + data: any + Any object that is serializable by cloudpickle (should be most things). + Will arrive as cloudpickled bytes in `.buffers[0]`. """ - serialized = cloudpickle.dumps(data) + import sys + if content is None: + content = {} + content['spyder_msg_type'] = spyder_msg_type self.session.send( self.iopub_socket, - 'spyder_data_msg', - buffers=[serialized], + 'spyder_msg', + content=content, + buffers=[cloudpickle.dumps(data)], + parent=self._parent_header, ) def get_value(self, name): @@ -157,14 +167,13 @@ def get_value(self, name): ns = self._get_current_namespace() value = ns[name] try: - self.publish_data({'spy_data': value}) + self.send_spyder_msg('spy_data', data=value) except: # * There is no need to inform users about # these errors. # * value = None makes Spyder to ignore # petitions to display a value - value = None - self.publish_data({'spy_data': value}) + self.send_spyder_msg('spy_data', data=None) self._do_publish_pdb_state = False def set_value(self, name, value): @@ -217,13 +226,13 @@ def save_namespace(self, filename): def publish_pdb_state(self): """ Publish Variable Explorer state and Pdb step through - publish_data. + send_spyder_msg. """ if self._pdb_obj and self._do_publish_pdb_state: state = dict(namespace_view = self.get_namespace_view(), var_properties = self.get_var_properties(), step = self._pdb_step) - self.publish_data({'spy_pdb_state': state}) + self.send_spyder_msg('spy_pdb_state', content={'pdb_state': state}) self._do_publish_pdb_state = True def pdb_continue(self): @@ -234,7 +243,7 @@ def pdb_continue(self): Fixes issue 2034 """ if self._pdb_obj: - self.publish_data({'spy_pdb_continue': True}) + self.send_spyder_msg('spy_pdb_continue') # --- For the Help plugin def is_defined(self, obj, force_import=False): diff --git a/spyder/widgets/ipythonconsole/namespacebrowser.py b/spyder/widgets/ipythonconsole/namespacebrowser.py index 977b17ad65d..4b0ce84f98c 100644 --- a/spyder/widgets/ipythonconsole/namespacebrowser.py +++ b/spyder/widgets/ipythonconsole/namespacebrowser.py @@ -15,7 +15,7 @@ import cloudpickle from qtconsole.rich_jupyter_widget import RichJupyterWidget -from spyder.config.base import _ +from spyder.config.base import _, debug_print from spyder.py3compat import to_text_string @@ -158,38 +158,32 @@ def save_namespace(self, filename): return self._kernel_reply # ---- Private API (defined by us) ------------------------------ - def _handle_spyder_data_msg(self, msg): + def _handle_spyder_msg(self, msg): """ - Handle spyder_data messages - - Content is cloudpickle-serialized in the buffers. + Handle internal spyder messages """ - # Deserialize data - try: - data = cloudpickle.loads(bytes(msg['buffers'][0])) - except Exception as msg: - self._kernel_value = None - self._kernel_reply = repr(msg) - self.sig_got_reply.emit() - return - - # Receive values asked for Spyder - value = data.get('spy_data', None) - if value is not None: - self._kernel_value = value + spyder_msg_type = msg['content'].get('spyder_msg_type') + if spyder_msg_type == 'spy_data': + # Deserialize data + try: + value = cloudpickle.loads(bytes(msg['buffers'][0])) + except Exception as msg: + self._kernel_value = None + self._kernel_reply = repr(msg) + else: + self._kernel_value = value self.sig_got_reply.emit() return - - # Receive Pdb state and dispatch it - pdb_state = data.get('spy_pdb_state', None) - if pdb_state is not None and isinstance(pdb_state, dict): - self.refresh_from_pdb(pdb_state) - - # Run Pdb continue to get to the first breakpoint - # Fixes 2034 - pdb_continue = data.get('spy_pdb_continue', None) - if pdb_continue: + elif spyder_msg_type == 'spy_pdb_state': + pdb_state = msg['content']['pdb_state'] + if pdb_state is not None and isinstance(pdb_state, dict): + self.refresh_from_pdb(pdb_state) + elif spyder_msg_type == 'spy_pdb_continue': + # Run Pdb continue to get to the first breakpoint + # Fixes 2034 self.write_to_stdin('continue') + else: + debug_print("No such spyder message type: %s" % spyder_msg_type) # ---- Private API (overrode by us) ---------------------------- def _handle_execute_reply(self, msg): From b015a5081e6917012f1b757a9064c9546da8d16c Mon Sep 17 00:00:00 2001 From: Min RK Date: Wed, 27 Sep 2017 13:33:53 +0200 Subject: [PATCH 03/16] appease overly pedantic linter --- spyder/utils/ipython/spyder_kernel.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spyder/utils/ipython/spyder_kernel.py b/spyder/utils/ipython/spyder_kernel.py index 904ee6400a4..b05e2b31af2 100644 --- a/spyder/utils/ipython/spyder_kernel.py +++ b/spyder/utils/ipython/spyder_kernel.py @@ -147,8 +147,8 @@ def send_spyder_msg(self, spyder_msg_type, content=None, data=None): content: dict The (JSONable) content of the message data: any - Any object that is serializable by cloudpickle (should be most things). - Will arrive as cloudpickled bytes in `.buffers[0]`. + Any object that is serializable by cloudpickle (should be most + things). Will arrive as cloudpickled bytes in `.buffers[0]`. """ import sys if content is None: From 82e0c030eac2206aaf81860f9f1f2ac92c6fb089 Mon Sep 17 00:00:00 2001 From: Min RK Date: Wed, 27 Sep 2017 13:43:43 +0200 Subject: [PATCH 04/16] install cloudpickle on ci that's 5 files now for every dependency - setup.py, requirements.txt, install.sh, one for each CI --- appveyor.yml | 2 +- continuous_integration/circle/install.sh | 2 +- continuous_integration/travis/install.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index ea30770afb1..c7ad5f2e8eb 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -11,7 +11,7 @@ environment: CONDA_DEPENDENCIES_FLAGS: "--quiet" CONDA_DEPENDENCIES: > rope pyflakes sphinx pygments pylint pycodestyle psutil nbconvert - qtawesome pickleshare pyzmq chardet mock pandas pytest + qtawesome cloudpickle pickleshare pyzmq chardet mock pandas pytest pytest-cov numpydoc scipy pillow qtconsole matplotlib jedi pywin32 PIP_DEPENDENCIES: "pytest-qt pytest-mock pytest-timeout flaky" diff --git a/continuous_integration/circle/install.sh b/continuous_integration/circle/install.sh index d18f8480da8..9e30aa83d1c 100755 --- a/continuous_integration/circle/install.sh +++ b/continuous_integration/circle/install.sh @@ -3,7 +3,7 @@ export CONDA_DEPENDENCIES_FLAGS="--quiet" export CONDA_DEPENDENCIES="rope pyflakes sphinx pygments pylint psutil nbconvert \ qtawesome pickleshare qtpy pyzmq chardet mock nomkl pandas \ - pytest pytest-cov numpydoc scipy cython pillow" + pytest pytest-cov numpydoc scipy cython pillow cloudpickle" export PIP_DEPENDENCIES="coveralls pytest-qt pytest-mock pytest-xvfb flaky jedi pycodestyle" # Download and install miniconda and conda/pip dependencies diff --git a/continuous_integration/travis/install.sh b/continuous_integration/travis/install.sh index 394abfc993c..e88b372ecd9 100755 --- a/continuous_integration/travis/install.sh +++ b/continuous_integration/travis/install.sh @@ -8,7 +8,7 @@ if [ "$TRAVIS_PYTHON_VERSION" = "3.5" ] && [ "$USE_PYQT" = "pyqt5" ]; then else export CONDA_DEPENDENCIES_FLAGS="--quiet" export CONDA_DEPENDENCIES="rope pyflakes sphinx pygments pylint psutil nbconvert \ - qtawesome pickleshare qtpy pyzmq chardet mock nomkl pandas \ + qtawesome cloudpickle pickleshare qtpy pyzmq chardet mock nomkl pandas \ pytest pytest-cov numpydoc scipy cython pillow jedi pycodestyle" export PIP_DEPENDENCIES="coveralls pytest-qt pytest-mock pytest-timeout flaky" fi From 2e053d3278ee2f2af565c1f36a6d2eeda8e8be93 Mon Sep 17 00:00:00 2001 From: Carlos Cordoba Date: Sun, 26 Nov 2017 23:23:57 -0500 Subject: [PATCH 05/16] Port Min's work to the 3.x branch --- .../variableexplorer/collectionseditor.py | 1 - .../variableexplorer/namespacebrowser.py | 18 ++---------------- 2 files changed, 2 insertions(+), 17 deletions(-) diff --git a/spyder/widgets/variableexplorer/collectionseditor.py b/spyder/widgets/variableexplorer/collectionseditor.py index 5bf63cad590..4e438dae3ea 100644 --- a/spyder/widgets/variableexplorer/collectionseditor.py +++ b/spyder/widgets/variableexplorer/collectionseditor.py @@ -24,7 +24,6 @@ import warnings # Third party imports -import cloudpickle from qtpy.compat import getsavefilename, to_qvariant from qtpy.QtCore import (QAbstractTableModel, QDateTime, QModelIndex, Qt, Signal, Slot) diff --git a/spyder/widgets/variableexplorer/namespacebrowser.py b/spyder/widgets/variableexplorer/namespacebrowser.py index d010637245a..eff4ba74476 100644 --- a/spyder/widgets/variableexplorer/namespacebrowser.py +++ b/spyder/widgets/variableexplorer/namespacebrowser.py @@ -21,11 +21,7 @@ QMessageBox, QToolButton, QVBoxLayout, QWidget) # Third party imports (others) -try: - import ipykernel.pickleutil - from ipykernel.serialize import serialize_object -except ImportError: - serialize_object = None +import cloudpickle # Local imports from spyder.config.base import _, get_supported_types @@ -46,16 +42,6 @@ SUPPORTED_TYPES = get_supported_types() -# XXX --- Disable canning for Numpy arrays for now --- -# This allows getting values between a Python 3 frontend -# and a Python 2 kernel, and viceversa, for several types of -# arrays. -# See this link for interesting ideas on how to solve this -# in the future: -# http://stackoverflow.com/q/30698004/438386 -if serialize_object is not None: - ipykernel.pickleutil.can_map.pop('numpy.ndarray') - class NamespaceBrowser(QWidget): """Namespace browser (global variables explorer widget)""" @@ -302,7 +288,7 @@ def get_value(self, name): def set_value(self, name, value): """Set value for a variable.""" try: - value = serialize_object(value) + value = cloudpickle.dumps(value) self.shellwidget.set_value(name, value) except TypeError as e: QMessageBox.critical(self, _("Error"), From 756494ad1a5616ae3ac7a71aab3a7c7f50996d77 Mon Sep 17 00:00:00 2001 From: Carlos Cordoba Date: Wed, 29 Nov 2017 12:25:03 -0500 Subject: [PATCH 06/16] IPython console: Fix loading data sent from the kernel in Python 2 --- spyder/widgets/ipythonconsole/namespacebrowser.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/spyder/widgets/ipythonconsole/namespacebrowser.py b/spyder/widgets/ipythonconsole/namespacebrowser.py index 4b0ce84f98c..0a6501472b7 100644 --- a/spyder/widgets/ipythonconsole/namespacebrowser.py +++ b/spyder/widgets/ipythonconsole/namespacebrowser.py @@ -16,7 +16,7 @@ from qtconsole.rich_jupyter_widget import RichJupyterWidget from spyder.config.base import _, debug_print -from spyder.py3compat import to_text_string +from spyder.py3compat import PY2, to_text_string class NamepaceBrowserWidget(RichJupyterWidget): @@ -166,7 +166,10 @@ def _handle_spyder_msg(self, msg): if spyder_msg_type == 'spy_data': # Deserialize data try: - value = cloudpickle.loads(bytes(msg['buffers'][0])) + if PY2: + value = cloudpickle.loads(msg['buffers'][0]) + else: + value = cloudpickle.loads(bytes(msg['buffers'][0])) except Exception as msg: self._kernel_value = None self._kernel_reply = repr(msg) From 8be884ab78747a5a7e73c608bc11090cd463fb85 Mon Sep 17 00:00:00 2001 From: Carlos Cordoba Date: Wed, 29 Nov 2017 12:26:02 -0500 Subject: [PATCH 07/16] IPython kernel: Remove unneeded import --- spyder/utils/ipython/spyder_kernel.py | 1 - 1 file changed, 1 deletion(-) diff --git a/spyder/utils/ipython/spyder_kernel.py b/spyder/utils/ipython/spyder_kernel.py index b05e2b31af2..0fabab1a43e 100644 --- a/spyder/utils/ipython/spyder_kernel.py +++ b/spyder/utils/ipython/spyder_kernel.py @@ -150,7 +150,6 @@ def send_spyder_msg(self, spyder_msg_type, content=None, data=None): Any object that is serializable by cloudpickle (should be most things). Will arrive as cloudpickled bytes in `.buffers[0]`. """ - import sys if content is None: content = {} content['spyder_msg_type'] = spyder_msg_type From 4dfa752a8ecd9b2ee82e20cdcfb180ce0f0c322b Mon Sep 17 00:00:00 2001 From: Carlos Cordoba Date: Wed, 29 Nov 2017 12:50:52 -0500 Subject: [PATCH 08/16] IPython console: Set used pickle protocol to 2 to guarantee interoperability between PY2 and 3 --- spyder/app/tests/test_mainwindow.py | 3 ++- spyder/plugins/tests/test_ipythonconsole.py | 4 ++-- spyder/utils/ipython/spyder_kernel.py | 5 ++++- spyder/widgets/variableexplorer/namespacebrowser.py | 7 +++++-- 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/spyder/app/tests/test_mainwindow.py b/spyder/app/tests/test_mainwindow.py index b05b446c051..06f46695d3f 100644 --- a/spyder/app/tests/test_mainwindow.py +++ b/spyder/app/tests/test_mainwindow.py @@ -397,7 +397,8 @@ def test_np_threshold(main_window, qtbot): @flaky(max_runs=3) -@pytest.mark.skipif(os.name == 'nt', reason="It times out sometimes on Windows") +@pytest.mark.skipif(os.name == 'nt' or (PY2 and PYQT4), + reason="It times out sometimes on Windows and fails in PY2 and PyQt4") def test_change_types_in_varexp(main_window, qtbot): """Test that variable types can't be changed in the Variable Explorer.""" # Create object diff --git a/spyder/plugins/tests/test_ipythonconsole.py b/spyder/plugins/tests/test_ipythonconsole.py index 0e6c54e1818..949e5508035 100644 --- a/spyder/plugins/tests/test_ipythonconsole.py +++ b/spyder/plugins/tests/test_ipythonconsole.py @@ -323,7 +323,7 @@ def test_unicode_vars(ipyconsole, qtbot): assert shell.get_value('д') == 10 # Change its value and verify - shell.set_value('д', cloudpickle.dumps(20)) + shell.set_value('д', cloudpickle.dumps(20, protocol=2)) qtbot.wait(1000) assert shell.get_value('д') == 20 @@ -373,7 +373,7 @@ def test_values_dbg(ipyconsole, qtbot): assert shell.get_value('aa') == 10 # Set value - shell.set_value('aa', cloudpickle.dumps(20)) + shell.set_value('aa', cloudpickle.dumps(20, protocol=2)) qtbot.wait(1000) assert shell.get_value('aa') == 20 diff --git a/spyder/utils/ipython/spyder_kernel.py b/spyder/utils/ipython/spyder_kernel.py index 0fabab1a43e..175f38aa0da 100644 --- a/spyder/utils/ipython/spyder_kernel.py +++ b/spyder/utils/ipython/spyder_kernel.py @@ -44,6 +44,9 @@ # shown at all there) EXCLUDED_NAMES = ['In', 'Out', 'exit', 'get_ipython', 'quit'] +# To be able to get and set variables between Python 2 and 3 +PICKLE_PROTOCOL = 2 + class SpyderKernel(IPythonKernel): """Spyder kernel for Jupyter""" @@ -157,7 +160,7 @@ def send_spyder_msg(self, spyder_msg_type, content=None, data=None): self.iopub_socket, 'spyder_msg', content=content, - buffers=[cloudpickle.dumps(data)], + buffers=[cloudpickle.dumps(data, protocol=PICKLE_PROTOCOL)], parent=self._parent_header, ) diff --git a/spyder/widgets/variableexplorer/namespacebrowser.py b/spyder/widgets/variableexplorer/namespacebrowser.py index eff4ba74476..3b38e6be218 100644 --- a/spyder/widgets/variableexplorer/namespacebrowser.py +++ b/spyder/widgets/variableexplorer/namespacebrowser.py @@ -42,6 +42,9 @@ SUPPORTED_TYPES = get_supported_types() +# To be able to get and set variables between Python 2 and 3 +PICKLE_PROTOCOL = 2 + class NamespaceBrowser(QWidget): """Namespace browser (global variables explorer widget)""" @@ -288,8 +291,8 @@ def get_value(self, name): def set_value(self, name, value): """Set value for a variable.""" try: - value = cloudpickle.dumps(value) - self.shellwidget.set_value(name, value) + svalue = cloudpickle.dumps(value, protocol=PICKLE_PROTOCOL) + self.shellwidget.set_value(name, svalue) except TypeError as e: QMessageBox.critical(self, _("Error"), "TypeError: %s" % to_text_string(e)) From e473bbda16efa1870e366cbd567227c2c135f17f Mon Sep 17 00:00:00 2001 From: Carlos Cordoba Date: Wed, 29 Nov 2017 13:17:06 -0500 Subject: [PATCH 09/16] Sitecustomize: Remove code to filter warning in ipykernel.datapub --- spyder/utils/site/sitecustomize.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/spyder/utils/site/sitecustomize.py b/spyder/utils/site/sitecustomize.py index a495e3ec4e6..7cd3ff57df9 100644 --- a/spyder/utils/site/sitecustomize.py +++ b/spyder/utils/site/sitecustomize.py @@ -280,13 +280,6 @@ def __init__(self, *args, **kwargs): TestProgram.__init__(self, *args, **kwargs) unittest.main = IPyTesProgram -# Filter warnings that appear for ipykernel when interacting with -# the Variable explorer (i.e trying to see a variable) -# Fixes Issue 5591 -warnings.filterwarnings(action='ignore', category=DeprecationWarning, - module='ipykernel.datapub', - message=".*ipykernel.datapub is deprecated.*") - #============================================================================== # Pandas adjustments From 77120572d79946f781281ec7d82dd687934c14f0 Mon Sep 17 00:00:00 2001 From: Carlos Cordoba Date: Wed, 29 Nov 2017 13:27:29 -0500 Subject: [PATCH 10/16] IPython console: Inform users that cloudpickle needs to be installed on external kernels --- spyder/app/tests/test_mainwindow.py | 3 +-- spyder/plugins/ipythonconsole.py | 24 ++++++++++++++---------- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/spyder/app/tests/test_mainwindow.py b/spyder/app/tests/test_mainwindow.py index 06f46695d3f..b05b446c051 100644 --- a/spyder/app/tests/test_mainwindow.py +++ b/spyder/app/tests/test_mainwindow.py @@ -397,8 +397,7 @@ def test_np_threshold(main_window, qtbot): @flaky(max_runs=3) -@pytest.mark.skipif(os.name == 'nt' or (PY2 and PYQT4), - reason="It times out sometimes on Windows and fails in PY2 and PyQt4") +@pytest.mark.skipif(os.name == 'nt', reason="It times out sometimes on Windows") def test_change_types_in_varexp(main_window, qtbot): """Test that variable types can't be changed in the Variable Explorer.""" # Create object diff --git a/spyder/plugins/ipythonconsole.py b/spyder/plugins/ipythonconsole.py index 880df084599..c6ed205344f 100644 --- a/spyder/plugins/ipythonconsole.py +++ b/spyder/plugins/ipythonconsole.py @@ -994,20 +994,24 @@ def create_new_client(self, give_focus=True, filename=''): # Else we won't be able to create a client if not CONF.get('main_interpreter', 'default'): pyexec = CONF.get('main_interpreter', 'executable') - ipykernel_present = programs.is_module_installed('ipykernel', - interpreter=pyexec) - if not ipykernel_present: + has_ipykernel = programs.is_module_installed('ipykernel', + interpreter=pyexec) + has_cloudpickle = programs.is_module_installed('cloudpickle', + interpreter=pyexec) + if not (has_ipykernel and has_cloudpickle): client.show_kernel_error(_("Your Python environment or " "installation doesn't " - "have the ipykernel module " - "installed on it. Without this module is " - "not possible for Spyder to create a " + "have the ipykernel and " + "cloudpickle modules " + "installed on it. Without these modules " + "is not possible for Spyder to create a " "console for you.

" - "You can install ipykernel by " - "running in a terminal:

" - "pip install ipykernel

" + "You can install them by running " + "in a system terminal:

" + "pip install ipykernel cloudpickle" + "

" "or

" - "conda install ipykernel")) + "conda install ipykernel cloudpickle")) return self.connect_client_to_kernel(client) From 2ec165bcffbfae667d27393688ba3635c3cb40fc Mon Sep 17 00:00:00 2001 From: Carlos Cordoba Date: Wed, 29 Nov 2017 15:15:35 -0500 Subject: [PATCH 11/16] IPython console: Fix setting values in Python 2 --- spyder/utils/ipython/spyder_kernel.py | 10 ++++++++-- spyder/widgets/variableexplorer/namespacebrowser.py | 4 +++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/spyder/utils/ipython/spyder_kernel.py b/spyder/utils/ipython/spyder_kernel.py index 175f38aa0da..941df964c3d 100644 --- a/spyder/utils/ipython/spyder_kernel.py +++ b/spyder/utils/ipython/spyder_kernel.py @@ -181,8 +181,14 @@ def get_value(self, name): def set_value(self, name, value): """Set the value of a variable""" ns = self._get_reference_namespace(name) - value = cloudpickle.loads(value) - ns[name] = value + + # We send serialized values in a list of one element + # from Spyder to the kernel, to be able to send them + # at all in Python 2 + svalue = value[0] + + dvalue = cloudpickle.loads(svalue) + ns[name] = dvalue def remove_value(self, name): """Remove a variable""" diff --git a/spyder/widgets/variableexplorer/namespacebrowser.py b/spyder/widgets/variableexplorer/namespacebrowser.py index 3b38e6be218..14112a4ae91 100644 --- a/spyder/widgets/variableexplorer/namespacebrowser.py +++ b/spyder/widgets/variableexplorer/namespacebrowser.py @@ -291,7 +291,9 @@ def get_value(self, name): def set_value(self, name, value): """Set value for a variable.""" try: - svalue = cloudpickle.dumps(value, protocol=PICKLE_PROTOCOL) + # We need to enclose values in a list to be able to send + # them to the kernel in Python 2 + svalue = [cloudpickle.dumps(value, protocol=PICKLE_PROTOCOL)] self.shellwidget.set_value(name, svalue) except TypeError as e: QMessageBox.critical(self, _("Error"), From dd7d7694f3337ff843386a255b0db6a3b92f4c6d Mon Sep 17 00:00:00 2001 From: Carlos Cordoba Date: Wed, 29 Nov 2017 15:34:20 -0500 Subject: [PATCH 12/16] IPython console: Remove spy prefix from Spyder message types --- spyder/utils/ipython/spyder_kernel.py | 8 ++++---- spyder/widgets/ipythonconsole/namespacebrowser.py | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/spyder/utils/ipython/spyder_kernel.py b/spyder/utils/ipython/spyder_kernel.py index 941df964c3d..bb5e4603513 100644 --- a/spyder/utils/ipython/spyder_kernel.py +++ b/spyder/utils/ipython/spyder_kernel.py @@ -169,13 +169,13 @@ def get_value(self, name): ns = self._get_current_namespace() value = ns[name] try: - self.send_spyder_msg('spy_data', data=value) + self.send_spyder_msg('data', data=value) except: # * There is no need to inform users about # these errors. # * value = None makes Spyder to ignore # petitions to display a value - self.send_spyder_msg('spy_data', data=None) + self.send_spyder_msg('data', data=None) self._do_publish_pdb_state = False def set_value(self, name, value): @@ -240,7 +240,7 @@ def publish_pdb_state(self): state = dict(namespace_view = self.get_namespace_view(), var_properties = self.get_var_properties(), step = self._pdb_step) - self.send_spyder_msg('spy_pdb_state', content={'pdb_state': state}) + self.send_spyder_msg('pdb_state', content={'pdb_state': state}) self._do_publish_pdb_state = True def pdb_continue(self): @@ -251,7 +251,7 @@ def pdb_continue(self): Fixes issue 2034 """ if self._pdb_obj: - self.send_spyder_msg('spy_pdb_continue') + self.send_spyder_msg('pdb_continue') # --- For the Help plugin def is_defined(self, obj, force_import=False): diff --git a/spyder/widgets/ipythonconsole/namespacebrowser.py b/spyder/widgets/ipythonconsole/namespacebrowser.py index 0a6501472b7..28a7b9ab2be 100644 --- a/spyder/widgets/ipythonconsole/namespacebrowser.py +++ b/spyder/widgets/ipythonconsole/namespacebrowser.py @@ -163,7 +163,7 @@ def _handle_spyder_msg(self, msg): Handle internal spyder messages """ spyder_msg_type = msg['content'].get('spyder_msg_type') - if spyder_msg_type == 'spy_data': + if spyder_msg_type == 'data': # Deserialize data try: if PY2: @@ -177,11 +177,11 @@ def _handle_spyder_msg(self, msg): self._kernel_value = value self.sig_got_reply.emit() return - elif spyder_msg_type == 'spy_pdb_state': + elif spyder_msg_type == 'pdb_state': pdb_state = msg['content']['pdb_state'] if pdb_state is not None and isinstance(pdb_state, dict): self.refresh_from_pdb(pdb_state) - elif spyder_msg_type == 'spy_pdb_continue': + elif spyder_msg_type == 'pdb_continue': # Run Pdb continue to get to the first breakpoint # Fixes 2034 self.write_to_stdin('continue') From 027e278b9b6986c86c558ff25291553ebd6be15e Mon Sep 17 00:00:00 2001 From: Carlos Cordoba Date: Wed, 29 Nov 2017 15:38:43 -0500 Subject: [PATCH 13/16] Testing: Fix a couple of tests in Python 3 --- spyder/plugins/tests/test_ipythonconsole.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spyder/plugins/tests/test_ipythonconsole.py b/spyder/plugins/tests/test_ipythonconsole.py index 949e5508035..3fd0cc94e1a 100644 --- a/spyder/plugins/tests/test_ipythonconsole.py +++ b/spyder/plugins/tests/test_ipythonconsole.py @@ -323,7 +323,7 @@ def test_unicode_vars(ipyconsole, qtbot): assert shell.get_value('д') == 10 # Change its value and verify - shell.set_value('д', cloudpickle.dumps(20, protocol=2)) + shell.set_value('д', [cloudpickle.dumps(20, protocol=2)]) qtbot.wait(1000) assert shell.get_value('д') == 20 @@ -373,7 +373,7 @@ def test_values_dbg(ipyconsole, qtbot): assert shell.get_value('aa') == 10 # Set value - shell.set_value('aa', cloudpickle.dumps(20, protocol=2)) + shell.set_value('aa', [cloudpickle.dumps(20, protocol=2)]) qtbot.wait(1000) assert shell.get_value('aa') == 20 From 86bfe8a16ad55f0b355805218e7e2fcfe68d810d Mon Sep 17 00:00:00 2001 From: Carlos Cordoba Date: Wed, 29 Nov 2017 18:46:39 -0500 Subject: [PATCH 14/16] Conda recipe: Add cloudpickle as a run dep --- conda.recipe/meta.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index b85d38eeb12..8cb36d15809 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -33,6 +33,7 @@ requirements: - pyzmq - chardet >=2.0.0 - numpydoc + - cloudpickle about: home: https://github.com/spyder-ide/spyder From 90a3b9b69905fa685601bc67b0869d00e400db91 Mon Sep 17 00:00:00 2001 From: Carlos Cordoba Date: Wed, 29 Nov 2017 19:23:00 -0500 Subject: [PATCH 15/16] IPython console: Convert serialized values to bytes when the frontend runs in PY2 and the kernel in PY3 --- spyder/utils/ipython/spyder_kernel.py | 11 ++++++++++- spyder/widgets/ipythonconsole/namespacebrowser.py | 4 +++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/spyder/utils/ipython/spyder_kernel.py b/spyder/utils/ipython/spyder_kernel.py index bb5e4603513..2decc9baeea 100644 --- a/spyder/utils/ipython/spyder_kernel.py +++ b/spyder/utils/ipython/spyder_kernel.py @@ -40,6 +40,9 @@ make_remote_view) +PY2 = sys.version[0] == '2' + + # Excluded variables from the Variable Explorer (i.e. they are not # shown at all there) EXCLUDED_NAMES = ['In', 'Out', 'exit', 'get_ipython', 'quit'] @@ -178,7 +181,7 @@ def get_value(self, name): self.send_spyder_msg('data', data=None) self._do_publish_pdb_state = False - def set_value(self, name, value): + def set_value(self, name, value, PY2_frontend): """Set the value of a variable""" ns = self._get_reference_namespace(name) @@ -187,6 +190,12 @@ def set_value(self, name, value): # at all in Python 2 svalue = value[0] + # We need to convert svalue to bytes if the frontend + # runs in Python 2 and the kernel runs in Python 3 + if PY2_frontend and not PY2: + svalue = bytes(svalue, 'latin-1') + + # Deserialize and set value in namespace dvalue = cloudpickle.loads(svalue) ns[name] = dvalue diff --git a/spyder/widgets/ipythonconsole/namespacebrowser.py b/spyder/widgets/ipythonconsole/namespacebrowser.py index 28a7b9ab2be..39f5af24a65 100644 --- a/spyder/widgets/ipythonconsole/namespacebrowser.py +++ b/spyder/widgets/ipythonconsole/namespacebrowser.py @@ -97,7 +97,9 @@ def get_value(self, name): def set_value(self, name, value): """Set value for a variable""" value = to_text_string(value) - code = u"get_ipython().kernel.set_value('%s', %s)" % (name, value) + code = u"get_ipython().kernel.set_value('%s', %s, %s)" % (name, value, + PY2) + if self._reading: self.kernel_client.input(u'!' + code) else: From 33a88db303008057e45c0718e2c70973933c63af Mon Sep 17 00:00:00 2001 From: Carlos Cordoba Date: Wed, 29 Nov 2017 19:38:26 -0500 Subject: [PATCH 16/16] IPython kernel: Fix setting the selected Python interpreter to start kernels --- spyder/utils/ipython/kernelspec.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/spyder/utils/ipython/kernelspec.py b/spyder/utils/ipython/kernelspec.py index e244bc2970a..32b071a3cde 100644 --- a/spyder/utils/ipython/kernelspec.py +++ b/spyder/utils/ipython/kernelspec.py @@ -24,7 +24,6 @@ class SpyderKernelSpec(KernelSpec): """Kernel spec for Spyder kernels""" - default_interpreter = CONF.get('main_interpreter', 'default') spy_path = get_module_source_path('spyder') def __init__(self, **kwargs): @@ -37,7 +36,7 @@ def __init__(self, **kwargs): def argv(self): """Command to start kernels""" # Python interpreter used to start kernels - if self.default_interpreter: + if CONF.get('main_interpreter', 'default'): pyexec = get_python_executable() else: # Avoid IPython adding the virtualenv on which Spyder is running