Merge pull request #5341 from minrk/cloudpickle

PR: Trade ipykernel.serialize for cloudpickle and remove use of publish_data

appveyor.yml

@@ -11,7 +11,7 @@ environment:
     CONDA_DEPENDENCIES_FLAGS: "--quiet"
     CONDA_DEPENDENCIES: >
       rope pyflakes sphinx pygments pylint pycodestyle psutil nbconvert
-      qtawesome pickleshare pyzmq chardet mock pandas pytest
+      qtawesome cloudpickle pickleshare pyzmq chardet mock pandas pytest
       pytest-cov numpydoc scipy pillow qtconsole matplotlib jedi pywin32
     PIP_DEPENDENCIES: "pytest-qt pytest-mock pytest-timeout flaky"
 

conda.recipe/meta.yaml

@@ -33,6 +33,7 @@ requirements:
     - pyzmq
     - chardet >=2.0.0
     - numpydoc
+    - cloudpickle
 
 about:
   home: https://github.com/spyder-ide/spyder

continuous_integration/circle/install.sh

@@ -3,7 +3,7 @@
 export CONDA_DEPENDENCIES_FLAGS="--quiet"
 export CONDA_DEPENDENCIES="rope pyflakes sphinx pygments pylint psutil nbconvert \
                            qtawesome pickleshare qtpy pyzmq chardet mock nomkl pandas \
-                           pytest pytest-cov numpydoc scipy cython pillow"
+                           pytest pytest-cov numpydoc scipy cython pillow cloudpickle"
 export PIP_DEPENDENCIES="coveralls pytest-qt pytest-mock pytest-xvfb flaky jedi pycodestyle"
 
 # Download and install miniconda and conda/pip dependencies

continuous_integration/travis/install.sh

@@ -8,7 +8,7 @@ if [ "$TRAVIS_PYTHON_VERSION" = "3.5" ] && [ "$USE_PYQT" = "pyqt5" ]; then
 else
     export CONDA_DEPENDENCIES_FLAGS="--quiet"
     export CONDA_DEPENDENCIES="rope pyflakes sphinx pygments pylint psutil nbconvert \
-                               qtawesome pickleshare qtpy pyzmq chardet mock nomkl pandas \
+                               qtawesome cloudpickle pickleshare qtpy pyzmq chardet mock nomkl pandas \
                                pytest pytest-cov numpydoc scipy cython pillow jedi pycodestyle"
     export PIP_DEPENDENCIES="coveralls pytest-qt pytest-mock pytest-timeout flaky"
 fi

requirements/requirements.txt

@@ -1,3 +1,4 @@
+cloudpickle
 rope>=0.9.4
 jedi>=0.9.0
 pyflakes

setup.py

@@ -272,6 +272,7 @@ def run(self):
     import setuptools     # analysis:ignore
 
 install_requires = [
+    'cloudpickle',
     'rope>=0.10.5',
     'jedi>=0.9.0',
     'pyflakes',

spyder/plugins/ipythonconsole.py

@@ -994,20 +994,24 @@ def create_new_client(self, give_focus=True, filename=''):
         # Else we won't be able to create a client
         if not CONF.get('main_interpreter', 'default'):
             pyexec = CONF.get('main_interpreter', 'executable')
-            ipykernel_present = programs.is_module_installed('ipykernel',
-                                                            interpreter=pyexec)
-            if not ipykernel_present:
+            has_ipykernel = programs.is_module_installed('ipykernel',
+                                                         interpreter=pyexec)
+            has_cloudpickle = programs.is_module_installed('cloudpickle',
+                                                           interpreter=pyexec)
+            if not (has_ipykernel and has_cloudpickle):
                 client.show_kernel_error(_("Your Python environment or "
                                      "installation doesn't "
-                                     "have the <tt>ipykernel</tt> module "
-                                     "installed on it. Without this module is "
-                                     "not possible for Spyder to create a "
+                                     "have the <tt>ipykernel</tt> and "
+                                     "<tt>cloudpickle</tt> modules "
+                                     "installed on it. Without these modules "
+                                     "is not possible for Spyder to create a "
                                      "console for you.<br><br>"
-                                     "You can install <tt>ipykernel</tt> by "
-                                     "running in a terminal:<br><br>"
-                                     "<tt>pip install ipykernel</tt><br><br>"
+                                     "You can install them by running "
+                                     "in a system terminal:<br><br>"
+                                     "<tt>pip install ipykernel cloudpickle</tt>"
+                                     "<br><br>"
                                      "or<br><br>"
-                                     "<tt>conda install ipykernel</tt>"))
+                                     "<tt>conda install ipykernel cloudpickle</tt>"))
                 return
 
         self.connect_client_to_kernel(client)

spyder/plugins/tests/test_ipythonconsole.py

@@ -11,8 +11,8 @@
 import tempfile
 from textwrap import dedent
 
+import cloudpickle
 from flaky import flaky
-from ipykernel.serialize import serialize_object
 from pygments.token import Name
 import pytest
 from qtpy import PYQT4, PYQT5, PYQT_VERSION
@@ -323,7 +323,7 @@ def test_unicode_vars(ipyconsole, qtbot):
     assert shell.get_value('д') == 10
 
     # Change its value and verify
-    shell.set_value('д', serialize_object(20))
+    shell.set_value('д', [cloudpickle.dumps(20, protocol=2)])
     qtbot.wait(1000)
     assert shell.get_value('д') == 20
 
@@ -373,7 +373,7 @@ def test_values_dbg(ipyconsole, qtbot):
     assert shell.get_value('aa') == 10
 
     # Set value
-    shell.set_value('aa', serialize_object(20))
+    shell.set_value('aa', [cloudpickle.dumps(20, protocol=2)])
     qtbot.wait(1000)
     assert shell.get_value('aa') == 20
 

spyder/utils/ipython/kernelspec.py

@@ -24,7 +24,6 @@
 class SpyderKernelSpec(KernelSpec):
     """Kernel spec for Spyder kernels"""
 
-    default_interpreter = CONF.get('main_interpreter', 'default')
     spy_path = get_module_source_path('spyder')
 
     def __init__(self, **kwargs):
@@ -37,7 +36,7 @@ def __init__(self, **kwargs):
     def argv(self):
         """Command to start kernels"""
         # Python interpreter used to start kernels
-        if self.default_interpreter:
+        if CONF.get('main_interpreter', 'default'):
             pyexec = get_python_executable()
         else:
             # Avoid IPython adding the virtualenv on which Spyder is running

spyder/utils/ipython/spyder_kernel.py

@@ -15,11 +15,8 @@
 import traceback
 
 # Third-party imports
-from ipykernel.datapub import publish_data
+import cloudpickle
 from ipykernel.ipkernel import IPythonKernel
-import ipykernel.pickleutil
-from ipykernel.pickleutil import CannedObject
-from ipykernel.serialize import deserialize_object
 
 # Check if we are running under an external interpreter
 IS_EXT_INTERPRETER = os.environ.get('EXTERNAL_INTERPRETER', '').lower() == "true"
@@ -43,20 +40,16 @@
                                                 make_remote_view)
 
 
-# XXX --- Disable canning for Numpy arrays for now ---
-# This allows getting values between a Python 3 frontend
-# and a Python 2 kernel, and viceversa, for several types of
-# arrays.
-# See this link for interesting ideas on how to solve this
-# in the future:
-# http://stackoverflow.com/q/30698004/438386
-ipykernel.pickleutil.can_map.pop('numpy.ndarray')
+PY2 = sys.version[0] == '2'
 
 
 # Excluded variables from the Variable Explorer (i.e. they are not
 # shown at all there)
 EXCLUDED_NAMES = ['In', 'Out', 'exit', 'get_ipython', 'quit']
 
+# To be able to get and set variables between Python 2 and 3
+PICKLE_PROTOCOL = 2
+
 
 class SpyderKernel(IPythonKernel):
     """Spyder kernel for Jupyter"""
@@ -149,28 +142,62 @@ def get_var_properties(self):
         else:
             return repr(None)
 
+    def send_spyder_msg(self, spyder_msg_type, content=None, data=None):
+        """publish custom messages to the spyder frontend
+
+        Parameters
+        ----------
+
+        spyder_msg_type: str
+            The spyder message type
+        content: dict
+            The (JSONable) content of the message
+        data: any
+            Any object that is serializable by cloudpickle (should be most
+            things). Will arrive as cloudpickled bytes in `.buffers[0]`.
+        """
+        if content is None:
+            content = {}
+        content['spyder_msg_type'] = spyder_msg_type
+        self.session.send(
+            self.iopub_socket,
+            'spyder_msg',
+            content=content,
+            buffers=[cloudpickle.dumps(data, protocol=PICKLE_PROTOCOL)],
+            parent=self._parent_header,
+        )
+
     def get_value(self, name):
         """Get the value of a variable"""
         ns = self._get_current_namespace()
         value = ns[name]
         try:
-            publish_data({'__spy_data__': value})
+            self.send_spyder_msg('data', data=value)
         except:
             # * There is no need to inform users about
             #   these errors.
             # * value = None makes Spyder to ignore
             #   petitions to display a value
-            value = None
-            publish_data({'__spy_data__': value})
+            self.send_spyder_msg('data', data=None)
         self._do_publish_pdb_state = False
 
-    def set_value(self, name, value):
+    def set_value(self, name, value, PY2_frontend):
         """Set the value of a variable"""
         ns = self._get_reference_namespace(name)
-        value = deserialize_object(value)[0]
-        if isinstance(value, CannedObject):
-            value = value.get_object()
-        ns[name] = value
+
+        # We send serialized values in a list of one element
+        # from Spyder to the kernel, to be able to send them
+        # at all in Python 2
+        svalue = value[0]
+
+        # We need to convert svalue to bytes if the frontend
+        # runs in Python 2 and the kernel runs in Python 3
+        if PY2_frontend and not PY2:
+            svalue = bytes(svalue, 'latin-1')
+
+        # Deserialize and set value in namespace
+        dvalue = cloudpickle.loads(svalue)
+        ns[name] = dvalue
 
     def remove_value(self, name):
         """Remove a variable"""
@@ -216,13 +243,13 @@ def save_namespace(self, filename):
     def publish_pdb_state(self):
         """
         Publish Variable Explorer state and Pdb step through
-        publish_data.
+        send_spyder_msg.
         """
         if self._pdb_obj and self._do_publish_pdb_state:
             state = dict(namespace_view = self.get_namespace_view(),
                          var_properties = self.get_var_properties(),
                          step = self._pdb_step)
-            publish_data({'__spy_pdb_state__': state})
+            self.send_spyder_msg('pdb_state', content={'pdb_state': state})
         self._do_publish_pdb_state = True
 
     def pdb_continue(self):
@@ -233,7 +260,7 @@ def pdb_continue(self):
         Fixes issue 2034
         """
         if self._pdb_obj:
-            publish_data({'__spy_pdb_continue__': True})
+            self.send_spyder_msg('pdb_continue')
 
     # --- For the Help plugin
     def is_defined(self, obj, force_import=False):

spyder/utils/site/sitecustomize.py

@@ -280,13 +280,6 @@ def __init__(self, *args, **kwargs):
         TestProgram.__init__(self, *args, **kwargs)
 unittest.main = IPyTesProgram
 
-# Filter warnings that appear for ipykernel when interacting with
-# the Variable explorer (i.e trying to see a variable)
-# Fixes Issue 5591
-warnings.filterwarnings(action='ignore', category=DeprecationWarning,
-                        module='ipykernel.datapub',
-                        message=".*ipykernel.datapub is deprecated.*")
-
 
 #==============================================================================
 # Pandas adjustments

spyder/widgets/ipythonconsole/namespacebrowser.py

@@ -12,12 +12,11 @@
 from qtpy.QtCore import QEventLoop
 from qtpy.QtWidgets import QMessageBox
 
-from ipykernel.pickleutil import CannedObject
-from ipykernel.serialize import deserialize_object
+import cloudpickle
 from qtconsole.rich_jupyter_widget import RichJupyterWidget
 
-from spyder.config.base import _
-from spyder.py3compat import to_text_string
+from spyder.config.base import _, debug_print
+from spyder.py3compat import PY2, to_text_string
 
 
 class NamepaceBrowserWidget(RichJupyterWidget):
@@ -98,7 +97,9 @@ def get_value(self, name):
     def set_value(self, name, value):
         """Set value for a variable"""
         value = to_text_string(value)
-        code = u"get_ipython().kernel.set_value('%s', %s)" % (name, value)
+        code = u"get_ipython().kernel.set_value('%s', %s, %s)" % (name, value,
+                                                                  PY2)
+
         if self._reading:
             self.kernel_client.input(u'!' + code)
         else:
@@ -159,41 +160,35 @@ def save_namespace(self, filename):
         return self._kernel_reply
 
     # ---- Private API (defined by us) ------------------------------
-    def _handle_data_message(self, msg):
+    def _handle_spyder_msg(self, msg):
         """
-        Handle raw (serialized) data sent by the kernel
-
-        We only handle data asked by Spyder, in case people use
-        publish_data for other purposes.
+        Handle internal spyder messages
         """
-        # Deserialize data
-        try:
-            data = deserialize_object(msg['buffers'])[0]
-        except Exception as msg:
-            self._kernel_value = None
-            self._kernel_reply = repr(msg)
-            self.sig_got_reply.emit()
-            return
-
-        # Receive values asked for Spyder
-        value = data.get('__spy_data__', None)
-        if value is not None:
-            if isinstance(value, CannedObject):
-                value = value.get_object()
-            self._kernel_value = value
+        spyder_msg_type = msg['content'].get('spyder_msg_type')
+        if spyder_msg_type == 'data':
+            # Deserialize data
+            try:
+                if PY2:
+                    value = cloudpickle.loads(msg['buffers'][0])
+                else:
+                    value = cloudpickle.loads(bytes(msg['buffers'][0]))
+            except Exception as msg:
+                self._kernel_value = None
+                self._kernel_reply = repr(msg)
+            else:
+                self._kernel_value = value
             self.sig_got_reply.emit()
             return
-
-        # Receive Pdb state and dispatch it
-        pdb_state = data.get('__spy_pdb_state__', None)
-        if pdb_state is not None and isinstance(pdb_state, dict):
-            self.refresh_from_pdb(pdb_state)
-
-        # Run Pdb continue to get to the first breakpoint
-        # Fixes 2034
-        pdb_continue = data.get('__spy_pdb_continue__', None)
-        if pdb_continue:
+        elif spyder_msg_type == 'pdb_state':
+            pdb_state = msg['content']['pdb_state']
+            if pdb_state is not None and isinstance(pdb_state, dict):
+                self.refresh_from_pdb(pdb_state)
+        elif spyder_msg_type == 'pdb_continue':
+            # Run Pdb continue to get to the first breakpoint
+            # Fixes 2034
             self.write_to_stdin('continue')
+        else:
+            debug_print("No such spyder message type: %s" % spyder_msg_type)
 
     # ---- Private API (overrode by us) ----------------------------
     def _handle_execute_reply(self, msg):