From cc7fe1e4e6ca0fe07c66c03f257784d82ed75563 Mon Sep 17 00:00:00 2001 From: Nik Haldimann Date: Tue, 31 Oct 2017 14:37:39 -0400 Subject: [PATCH 1/2] Restore compatibility with functions pickled with 0.4.0 Release 0.4.1 introduced a change to arguments of _fill_function which meant that functions pickled before that couldn't be unpickled anymore. Addresses https://github.com/cloudpipe/cloudpickle/issues/126 --- cloudpickle/cloudpickle.py | 12 +++++++++++- tests/cloudpickle_test.py | 21 +++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index e8c278020..1d9861d21 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -943,7 +943,17 @@ def __reduce__(cls): return cls.__name__ -def _fill_function(func, globals, defaults, dict, module, closure_values): +def _fill_function(*args): + if len(args) == 5: + # Backwards compat for cloudpickle v0.4.0, after which the `module` + # argument was introduced + updated_args = args[:-1] + (None, args[-1],) + return _fill_function_internal(*updated_args) + else: + return _fill_function_internal(*args) + + +def _fill_function_internal(func, globals, defaults, dict, module, closure_values): """ Fills in the rest of function data into the skeleton function object that were created via _make_skel_func(). """ diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py index 70424311d..77a8d3dca 100644 --- a/tests/cloudpickle_test.py +++ b/tests/cloudpickle_test.py @@ -724,6 +724,27 @@ def test_builtin_type__new__(self): for t in list, tuple, set, frozenset, dict, object: self.assertTrue(pickle_depickle(t.__new__) is t.__new__) + def test_function_pickle_compat_0_4_0(self): + # The result of `cloudpickle.dumps(lambda x: x)` in cloudpickle 0.4.0 + pickled = ('\x80\x02ccloudpickle.cloudpickle\n_fill_function\nq\x00(c' + 'cloudpickle.cloudpickle\n_make_skel_func\nq\x01ccloudpickle.clou' + 'dpickle\n_builtin_type\nq\x02U\x08CodeTypeq\x03\x85q\x04Rq\x05(K' + '\x01K\x01K\x01KCU\x04|\x00\x00Sq\x06N\x85q\x07)U\x01xq\x08\x85q' + '\tU\x07q\nU\x08q\x0bK\x01U\x00q\x0c))tq\rRq\x0eJ' + '\xff\xff\xff\xff}q\x0f\x87q\x10Rq\x11}q\x12N}q\x13NtR.') + self.assertEquals(42, cloudpickle.loads(pickled)(42)) + + def test_function_pickle_compat_0_4_1(self): + # The result of `cloudpickle.dumps(lambda x: x)` in cloudpickle 0.4.1 + pickled = ('\x80\x02ccloudpickle.cloudpickle\n_fill_function\nq\x00(c' + 'cloudpickle.cloudpickle\n_make_skel_func\nq\x01ccloudpickle.clou' + 'dpickle\n_builtin_type\nq\x02U\x08CodeTypeq\x03\x85q\x04Rq\x05(K' + '\x01K\x01K\x01KCU\x04|\x00\x00Sq\x06N\x85q\x07)U\x01xq\x08\x85q' + '\tU\x07q\nU\x08q\x0bK\x01U\x00q\x0c))tq\rRq\x0eJ' + '\xff\xff\xff\xff}q\x0f\x87q\x10Rq\x11}q\x12N}q\x13U\x08__main__q' + '\x14NtR.') + self.assertEquals(42, cloudpickle.loads(pickled)(42)) + if __name__ == '__main__': unittest.main() From 2b8809cca73f717c3817c4c4e9ca813202328347 Mon Sep 17 00:00:00 2001 From: Nik Haldimann Date: Tue, 31 Oct 2017 15:17:19 -0400 Subject: [PATCH 2/2] Skip tests for Python 3 as hardcoded pickle binary is incompatible I'm getting this error when unpickling with Python 3 (no stacktrace): ``` TypeError: an integer is required (got type str) ``` --- tests/cloudpickle_test.py | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py index 77a8d3dca..898e11ea4 100644 --- a/tests/cloudpickle_test.py +++ b/tests/cloudpickle_test.py @@ -724,25 +724,31 @@ def test_builtin_type__new__(self): for t in list, tuple, set, frozenset, dict, object: self.assertTrue(pickle_depickle(t.__new__) is t.__new__) + @pytest.mark.skipif(sys.version_info >= (3, 0), + reason="hardcoded pickle bytes for 2.7") def test_function_pickle_compat_0_4_0(self): - # The result of `cloudpickle.dumps(lambda x: x)` in cloudpickle 0.4.0 - pickled = ('\x80\x02ccloudpickle.cloudpickle\n_fill_function\nq\x00(c' - 'cloudpickle.cloudpickle\n_make_skel_func\nq\x01ccloudpickle.clou' - 'dpickle\n_builtin_type\nq\x02U\x08CodeTypeq\x03\x85q\x04Rq\x05(K' - '\x01K\x01K\x01KCU\x04|\x00\x00Sq\x06N\x85q\x07)U\x01xq\x08\x85q' - '\tU\x07q\nU\x08q\x0bK\x01U\x00q\x0c))tq\rRq\x0eJ' - '\xff\xff\xff\xff}q\x0f\x87q\x10Rq\x11}q\x12N}q\x13NtR.') + # The result of `cloudpickle.dumps(lambda x: x)` in cloudpickle 0.4.0, + # Python 2.7 + pickled = (b'\x80\x02ccloudpickle.cloudpickle\n_fill_function\nq\x00(c' + b'cloudpickle.cloudpickle\n_make_skel_func\nq\x01ccloudpickle.clou' + b'dpickle\n_builtin_type\nq\x02U\x08CodeTypeq\x03\x85q\x04Rq\x05(K' + b'\x01K\x01K\x01KCU\x04|\x00\x00Sq\x06N\x85q\x07)U\x01xq\x08\x85q' + b'\tU\x07q\nU\x08q\x0bK\x01U\x00q\x0c))tq\rRq\x0eJ' + b'\xff\xff\xff\xff}q\x0f\x87q\x10Rq\x11}q\x12N}q\x13NtR.') self.assertEquals(42, cloudpickle.loads(pickled)(42)) + @pytest.mark.skipif(sys.version_info >= (3, 0), + reason="hardcoded pickle bytes for 2.7") def test_function_pickle_compat_0_4_1(self): - # The result of `cloudpickle.dumps(lambda x: x)` in cloudpickle 0.4.1 - pickled = ('\x80\x02ccloudpickle.cloudpickle\n_fill_function\nq\x00(c' - 'cloudpickle.cloudpickle\n_make_skel_func\nq\x01ccloudpickle.clou' - 'dpickle\n_builtin_type\nq\x02U\x08CodeTypeq\x03\x85q\x04Rq\x05(K' - '\x01K\x01K\x01KCU\x04|\x00\x00Sq\x06N\x85q\x07)U\x01xq\x08\x85q' - '\tU\x07q\nU\x08q\x0bK\x01U\x00q\x0c))tq\rRq\x0eJ' - '\xff\xff\xff\xff}q\x0f\x87q\x10Rq\x11}q\x12N}q\x13U\x08__main__q' - '\x14NtR.') + # The result of `cloudpickle.dumps(lambda x: x)` in cloudpickle 0.4.1, + # Python 2.7 + pickled = (b'\x80\x02ccloudpickle.cloudpickle\n_fill_function\nq\x00(c' + b'cloudpickle.cloudpickle\n_make_skel_func\nq\x01ccloudpickle.clou' + b'dpickle\n_builtin_type\nq\x02U\x08CodeTypeq\x03\x85q\x04Rq\x05(K' + b'\x01K\x01K\x01KCU\x04|\x00\x00Sq\x06N\x85q\x07)U\x01xq\x08\x85q' + b'\tU\x07q\nU\x08q\x0bK\x01U\x00q\x0c))tq\rRq\x0eJ' + b'\xff\xff\xff\xff}q\x0f\x87q\x10Rq\x11}q\x12N}q\x13U\x08__main__q' + b'\x14NtR.') self.assertEquals(42, cloudpickle.loads(pickled)(42))