Pickle5 backport support for Python3.5

.gitignore

@@ -3,3 +3,4 @@ dist/
 *.egg-info/
 __pycache__/
 *.so
+.DS_Store

.travis.yml

@@ -13,6 +13,11 @@ env:
 
 matrix:
   include:
+  - python: "3.5"
+    dist: trusty
+    arch: amd64
+  - python: "3.5"
+    arch: amd64
   - python: "3.6"
     dist: trusty
     arch: amd64
@@ -27,6 +32,7 @@ matrix:
 
 install:
   - python -m venv $PY_VENV
+  - sudo chown -Rv $USER:$GROUP /home/travis/.cache/pip/wheels  # https://travis-ci.community/t/permission-issue-while-building-wheels-for-various-python-packages/7822/6
   - $PYTHON -m pip install -U pip wheel setuptools numpy
   - $PYTHON -m pip install -r requirements.txt
 

BACKPORT-NOTES.txt

@@ -12,3 +12,4 @@ then:
 
 * Same for 3.7
 
+For python3.5, we must use protocol 4 or greater to copy this object; since __getnewargs_ex__ returned keyword arguments.

patch

@@ -1,5 +1,5 @@
---- ../38/Include/picklebufobject.h	2019-11-02 16:20:20.358405065 +0100
-+++ pickle5/picklebufobject.h	2019-11-02 16:30:40.749731365 +0100
+--- ../38/Include/picklebufobject.h	2020-05-10 16:01:50.000000000 -0700
++++ pickle5/picklebufobject.h	2019-10-10 21:27:34.000000000 -0700
 @@ -10,18 +10,18 @@
 
  #ifndef Py_LIMITED_API
@@ -23,8 +23,8 @@
 
  #endif /* !Py_LIMITED_API */
 
---- ../38/Objects/picklebufobject.c	2019-11-02 16:20:20.834410181 +0100
-+++ pickle5/picklebufobject.c	2019-11-02 16:30:40.749731365 +0100
+--- ../38/Objects/picklebufobject.c	2020-05-10 16:01:50.000000000 -0700
++++ pickle5/picklebufobject.c	2019-10-10 21:29:15.000000000 -0700
 @@ -4,6 +4,8 @@
  #include "Python.h"
  #include <stddef.h>
@@ -34,8 +34,8 @@
  typedef struct {
      PyObject_HEAD
      /* The view exported by the original object */
---- ../38/Modules/_pickle.c	2019-11-02 16:20:20.754409320 +0100
-+++ pickle5/_pickle.c	2019-11-02 16:46:35.809907792 +0100
+--- ../38/Modules/_pickle.c	2020-05-10 16:01:50.000000000 -0700
++++ pickle5/_pickle.c	2020-02-14 17:54:56.000000000 -0800
 @@ -1,11 +1,11 @@
 -/* pickle accelerator C extensor: _pickle module.
 - *
@@ -55,11 +55,14 @@
  #endif
 
  #include "Python.h"
-@@ -709,7 +709,13 @@
+@@ -709,7 +709,16 @@
  static PyTypeObject Pickler_Type;
  static PyTypeObject Unpickler_Type;
 
-+#if PY_VERSION_HEX < 0x03070000
++
++#if PY_VERSION_HEX < 0x03060000
++#include "clinic/_pickle-3.5.c.h"
++#elif PY_VERSION_HEX < 0x03070000
 +#include "clinic/_pickle-3.6.c.h"
 +#elif PY_VERSION_HEX < 0x03080000
 +#include "clinic/_pickle-3.7.c.h"
@@ -69,7 +72,56 @@
 
  /*************************************************************************
   A custom hashtable mapping void* to Python ints. This is used by the pickler
-@@ -7020,11 +7026,6 @@
+@@ -4457,12 +4466,13 @@
+ dump(PicklerObject *self, PyObject *obj)
+ {
+     const char stop_op = STOP;
++    int status = -1;
+     PyObject *tmp;
+     _Py_IDENTIFIER(reducer_override);
+
+     if (_PyObject_LookupAttrId((PyObject *)self, &PyId_reducer_override,
+                                &tmp) < 0) {
+-        return -1;
++        goto error;
+     }
+     /* Cache the reducer_override method, if it exists. */
+     if (tmp != NULL) {
+@@ -4479,7 +4489,7 @@
+         assert(self->proto >= 0 && self->proto < 256);
+         header[1] = (unsigned char)self->proto;
+         if (_Pickler_Write(self, header, 2) < 0)
+-            return -1;
++            goto error;
+         if (self->proto >= 4)
+             self->framing = 1;
+     }
+@@ -4487,9 +4497,22 @@
+     if (save(self, obj, 0) < 0 ||
+         _Pickler_Write(self, &stop_op, 1) < 0 ||
+         _Pickler_CommitFrame(self) < 0)
+-        return -1;
++        goto error;
++
++    // Success
++    status = 0;
++
++  error:
+     self->framing = 0;
+-    return 0;
++
++    /* Break the reference cycle we generated at the beginning this function
++     * call when setting the reducer_override attribute of the Pickler instance
++     * to a bound method of the same instance. This is important as the Pickler
++     * instance holds a reference to each object it has pickled (through its
++     * memo): thus, these objects wont be garbage-collected as long as the
++     * Pickler itself is not collected. */
++    Py_CLEAR(self->reducer_override);
++    return status;
+ }
+
+ /*[clinic input]
+@@ -7020,11 +7043,6 @@
      PyObject *global;
      PyObject *module;
 
@@ -81,7 +133,7 @@
      /* Try to map the old names used in Python 2.x to the new ones used in
         Python 3.x.  We do this only with old pickle protocols and when the
         user has not disabled the feature. */
-@@ -7877,11 +7878,25 @@
+@@ -7877,11 +7895,25 @@
      return NULL;
  }
 
@@ -107,7 +159,7 @@
      {NULL, NULL} /* sentinel */
  };
 
-@@ -7965,6 +7980,8 @@
+@@ -7965,6 +7997,8 @@
      Py_INCREF(&Unpickler_Type);
      if (PyModule_AddObject(m, "Unpickler", (PyObject *)&Unpickler_Type) < 0)
          return NULL;
@@ -116,8 +168,8 @@
      Py_INCREF(&PyPickleBuffer_Type);
      if (PyModule_AddObject(m, "PickleBuffer",
                             (PyObject *)&PyPickleBuffer_Type) < 0)
---- ../38/Lib/pickle.py	2019-11-02 16:20:20.482406398 +0100
-+++ pickle5/pickle.py	2019-11-02 16:46:54.906114687 +0100
+--- ../38/Lib/pickle.py	2020-05-10 16:01:50.000000000 -0700
++++ pickle5/pickle.py	2019-12-07 22:59:41.000000000 -0800
 @@ -40,7 +40,7 @@
             "Unpickler", "dump", "dumps", "load", "loads"]
 
@@ -155,8 +207,8 @@
          PickleError,
          PicklingError,
          UnpicklingError,
---- ../38/Lib/pickletools.py	2019-11-02 16:20:20.486406441 +0100
-+++ pickle5/pickletools.py	2019-11-02 16:30:40.753731415 +0100
+--- ../38/Lib/pickletools.py	2020-05-10 16:01:50.000000000 -0700
++++ pickle5/pickletools.py	2019-10-10 22:43:59.000000000 -0700
 @@ -12,10 +12,11 @@
 
  import codecs
@@ -170,8 +222,8 @@
  __all__ = ['dis', 'genops', 'optimize']
 
  bytes_types = pickle.bytes_types
---- ../38/Lib/test/pickletester.py	2019-11-02 16:20:20.538407000 +0100
-+++ pickle5/test/pickletester.py	2019-11-02 16:42:03.558511125 +0100
+--- ../38/Lib/test/pickletester.py	2020-05-10 16:01:50.000000000 -0700
++++ pickle5/test/pickletester.py	2020-05-17 01:26:59.000000000 -0700
 @@ -5,8 +5,6 @@
  import functools
  import os
@@ -181,7 +233,46 @@
  import shutil
  import struct
  import sys
-@@ -32,7 +30,8 @@
+@@ -23,6 +21,38 @@
+
+ try:
+     import numpy as np
++
++    def _numpy_ndarray_reduce(array):
++        # This function is implemented according to 'array_reduce_ex_picklebuffer'
++        # in numpy C backend. This is a workaround for python3.5 pickling support.
++        if ((not array.flags.c_contiguous and not array.flags.f_contiguous) or
++                (issubclass(type(array), np.ndarray) and type(array) is not np.ndarray) or
++                array.dtype == "O" or array.itemsize == 0):
++            return array.__reduce__()
++
++        # This function is implemented according to 'array_reduce_ex_picklebuffer'
++        # in numpy C backend. This is a workaround for python3.5 pickling support.
++        from pickle5 import PickleBuffer as picklebuf_class
++
++        # if the array if Fortran-contiguous and not C-contiguous,
++        # the PickleBuffer instance will hold a view on the transpose
++        # of the initial array, that is C-contiguous.
++        if not array.flags.c_contiguous and array.flags.f_contiguous:
++            order = "F"
++            picklebuf_args = array.transpose()
++        else:
++            order = "C"
++            picklebuf_args = array
++        try:
++            buffer = picklebuf_class(picklebuf_args)
++        except Exception:
++            # Some arrays may refuse to export a buffer, in which case
++            # just fall back on regular __reduce_ex__ implementation
++            # (gh-12745).
++            return array.__reduce__()
++        from numpy.core.numeric import _frombuffer
++        return _frombuffer, (buffer, array.dtype, array.shape, order)
++
+ except ImportError:
+     np = None
+
+@@ -32,7 +62,8 @@
      _2G, _4G, bigmemtest, reap_threads, forget,
      )
 
@@ -191,7 +282,20 @@
 
  requires_32b = unittest.skipUnless(sys.maxsize < 2**32,
                                     "test is only meaningful on 32-bit builds")
-@@ -1445,12 +1444,11 @@
+@@ -273,9 +304,9 @@
+             return not (self == other)
+
+         def __repr__(self):
+-            return (f"{type(self)}(shape={self.array.shape},"
+-                    f"strides={self.array.strides}, "
+-                    f"bytes={self.array.tobytes()})")
++            return ("{}(shape={}, strides={}, bytes={})".format(
++                type(self), self.array.shape, self.array.strides,
++                self.array.tobytes()))
+
+         def __reduce_ex__(self, protocol):
+             if not self.array.contiguous:
+@@ -1445,12 +1476,11 @@
      # of 1.
      def dont_test_disassembly(self):
          from io import StringIO
@@ -205,24 +309,118 @@
              got = filelike.getvalue()
              self.assertEqual(expected, got)
 
-@@ -2328,7 +2326,6 @@
+@@ -2095,6 +2125,10 @@
+         x = ComplexNewObjEx.__new__(ComplexNewObjEx, 0xface)  # avoid __init__
+         x.abc = 666
+         for proto in protocols:
++            if sys.version_info < (3, 6) and proto < 4:
++                # '__getnewargs_ex__' is not supported in protocol 2 & 3 before Python3.6.
++                # See https://docs.python.org/3/library/pickle.html#object.__getnewargs_ex__
++                continue
+             with self.subTest(proto=proto):
+                 s = self.dumps(x, proto)
+                 if proto < 1:
+@@ -2328,7 +2362,6 @@
          elif frameless_start is not None:
              self.assertLess(pos - frameless_start, self.FRAME_SIZE_MIN)
 
 -    @support.skip_if_pgo_task
      def test_framing_many_objects(self):
          obj = list(range(10**5))
          for proto in range(4, pickle.HIGHEST_PROTOCOL + 1):
-@@ -2418,7 +2415,6 @@
+@@ -2418,7 +2451,6 @@
                                  count_opcode(pickle.FRAME, pickled))
                  self.assertEqual(obj, self.loads(some_frames_pickle))
 
 -    @support.skip_if_pgo_task
      def test_framed_write_sizes_with_delayed_writer(self):
          class ChunkAccumulator:
              """Accumulate pickler output in a list of raw chunks."""
---- ../38/Lib/test/test_pickle.py	2019-11-02 16:20:20.602407687 +0100
-+++ pickle5/test/test_pickle.py	2019-11-02 16:37:43.523126691 +0100
+@@ -2774,6 +2806,12 @@
+
+     @unittest.skipIf(np is None, "Test needs Numpy")
+     def test_buffers_numpy(self):
++        def _reduce(items):
++            base_obj = items[0](*items[1])
++            if len(items) > 2:
++                base_obj.__setstate__(items[2])
++            return base_obj
++
+         def check_no_copy(x, y):
+             np.testing.assert_equal(x, y)
+             self.assertEqual(x.ctypes.data, y.ctypes.data)
+@@ -2785,20 +2823,32 @@
+         def check_array(arr):
+             # In-band
+             for proto in range(0, pickle.HIGHEST_PROTOCOL + 1):
+-                data = self.dumps(arr, proto)
+-                new = self.loads(data)
++                if sys.version_info < (3, 6) and proto >= 5:
++                    data = self.dumps(_numpy_ndarray_reduce(arr), proto)
++                    new = _reduce(self.loads(data))
++                else:
++                    data = self.dumps(arr, proto)
++                    new = self.loads(data)
+                 check_copy(arr, new)
+             for proto in range(5, pickle.HIGHEST_PROTOCOL + 1):
+                 buffer_callback = lambda _: True
+-                data = self.dumps(arr, proto, buffer_callback=buffer_callback)
+-                new = self.loads(data)
++                if sys.version_info < (3, 6):
++                    data = self.dumps(_numpy_ndarray_reduce(arr), proto, buffer_callback=buffer_callback)
++                    new = _reduce(self.loads(data))
++                else:
++                    data = self.dumps(arr, proto, buffer_callback=buffer_callback)
++                    new = self.loads(data)
+                 check_copy(arr, new)
+             # Out-of-band
+             for proto in range(5, pickle.HIGHEST_PROTOCOL + 1):
+                 buffers = []
+                 buffer_callback = buffers.append
+-                data = self.dumps(arr, proto, buffer_callback=buffer_callback)
+-                new = self.loads(data, buffers=buffers)
++                if sys.version_info < (3, 6):
++                    data = self.dumps(_numpy_ndarray_reduce(arr), proto, buffer_callback=buffer_callback)
++                    new = _reduce(self.loads(data, buffers=buffers))
++                else:
++                    data = self.dumps(arr, proto, buffer_callback=buffer_callback)
++                    new = self.loads(data, buffers=buffers)
+                 if arr.flags.c_contiguous or arr.flags.f_contiguous:
+                     check_no_copy(arr, new)
+                 else:
+@@ -3497,6 +3547,30 @@
+                         ValueError, 'The reducer just failed'):
+                     p.dump(h)
+
++    @support.cpython_only
++    def test_reducer_override_no_reference_cycle(self):
++        # bpo-39492: reducer_override used to induce a spurious reference cycle
++        # inside the Pickler object, that could prevent all serialized objects
++        # from being garbage-collected without explicity invoking gc.collect.
++
++        for proto in range(0, pickle.HIGHEST_PROTOCOL + 1):
++            with self.subTest(proto=proto):
++                def f():
++                    pass
++
++                wr = weakref.ref(f)
++
++                bio = io.BytesIO()
++                p = self.pickler_class(bio, proto)
++                p.dump(f)
++                new_f = pickle.loads(bio.getvalue())
++                assert new_f == 5
++
++                del p
++                del f
++
++                self.assertIsNone(wr())
++
+
+ class AbstractDispatchTableTests(unittest.TestCase):
+
+--- ../38/Lib/test/test_pickle.py	2020-05-10 16:01:50.000000000 -0700
++++ pickle5/test/test_pickle.py	2020-05-16 22:15:47.000000000 -0700
 @@ -1,7 +1,6 @@
  from _compat_pickle import (IMPORT_MAPPING, REVERSE_IMPORT_MAPPING,
                              NAME_MAPPING, REVERSE_NAME_MAPPING)
@@ -272,8 +470,8 @@
 
      class CUnpicklerTests(PyUnpicklerTests):
          unpickler = _pickle.Unpickler
---- ../38/Lib/test/test_picklebuffer.py	2019-11-02 16:20:20.602407687 +0100
-+++ pickle5/test/test_picklebuffer.py	2019-11-02 16:38:22.283628888 +0100
+--- ../38/Lib/test/test_picklebuffer.py	2020-05-10 16:01:50.000000000 -0700
++++ pickle5/test/test_picklebuffer.py	2019-12-07 22:59:41.000000000 -0800
 @@ -4,12 +4,13 @@
  """