From 7c5bef7ec0af6f0f25e1b8ac7eacfd369ac8d0cf Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Thu, 7 Mar 2019 20:42:13 +0100 Subject: [PATCH 1/6] FIX fix relative import inside depickled functions --- cloudpickle/cloudpickle.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index 75c29977e..83352936b 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -666,6 +666,13 @@ def extract_func_data(self, func): # multiple invokations are bound to the same Cloudpickler. base_globals = self.globals_ref.setdefault(id(func.__globals__), {}) + if base_globals == {}: + # Add module attributes used to resolve relative imports + # instructions inside func. + for k in ["__package__", "__name__", "__path__", "__file__"]: + if k in func.__globals__: + base_globals[k] = func.__globals__[k] + return (code, f_globals, defaults, closure, dct, base_globals) def save_builtin_function(self, obj): From 7842fdd3fedab72ac5c0ab73784d3c62bf5e9ff4 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Thu, 7 Mar 2019 20:38:16 +0100 Subject: [PATCH 2/6] TST add a toy module for testing purposes --- tests/mypkg/__init__.py | 6 ++++++ tests/mypkg/mod.py | 2 ++ 2 files changed, 8 insertions(+) create mode 100644 tests/mypkg/__init__.py create mode 100644 tests/mypkg/mod.py diff --git a/tests/mypkg/__init__.py b/tests/mypkg/__init__.py new file mode 100644 index 000000000..fe3cc6b1d --- /dev/null +++ b/tests/mypkg/__init__.py @@ -0,0 +1,6 @@ +from .mod import module_function + + +def package_function(): + """Function living inside a package, not a simple module""" + return "hello from a package!" diff --git a/tests/mypkg/mod.py b/tests/mypkg/mod.py new file mode 100644 index 000000000..a703a6aa5 --- /dev/null +++ b/tests/mypkg/mod.py @@ -0,0 +1,2 @@ +def module_function(): + return "hello from a module!" From 6377d8cd746728d72ba37daab86abbd5472404de Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Thu, 7 Mar 2019 20:49:17 +0100 Subject: [PATCH 3/6] TST test relative import inside depickled function --- tests/cloudpickle_test.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py index 6ffe2ad0c..caa222ed3 100644 --- a/tests/cloudpickle_test.py +++ b/tests/cloudpickle_test.py @@ -1368,6 +1368,30 @@ def test_dataclass(self): pickle_depickle(DataClass, protocol=self.protocol) assert data.x == pickle_depickle(data, protocol=self.protocol).x == 42 + def test_relative_import_inside_function(self): + # Make sure relative imports inside round-tripped functions is not + # broken.This was a bug in cloudpickle versions <= 0.5.3 and was + # re-introduced in 0.8.0. + + # Both functions living inside modules and packages are tested. + def f(): + # module_function belongs to mypkg.mod1, which is a module + from .mypkg import module_function + return module_function() + + def g(): + # package_function belongs to mypkg, which is a package + from .mypkg import package_function + return package_function() + + for func, source in zip([f, g], ["module", "package"]): + # Make sure relative imports are initially working + assert func() == "hello from a {}!".format(source) + + # Make sure relative imports still work after round-tripping + cloned_func = pickle_depickle(func, protocol=self.protocol) + assert cloned_func() == "hello from a {}!".format(source) + class Protocol2CloudPickleTest(CloudPickleTest): From fb66cf51412f803cfa3180ac172e75411edffa05 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Mon, 11 Mar 2019 11:18:00 +0100 Subject: [PATCH 4/6] FIX handle case where __globals__ is None on PyPy --- cloudpickle/cloudpickle.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index 83352936b..b5f414ac2 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -670,7 +670,9 @@ def extract_func_data(self, func): # Add module attributes used to resolve relative imports # instructions inside func. for k in ["__package__", "__name__", "__path__", "__file__"]: - if k in func.__globals__: + # Some built-in functions/methods such as object.__new__ have + # their __globals__ set to None in PyPy + if func.__globals__ is not None and k in func.__globals__: base_globals[k] = func.__globals__[k] return (code, f_globals, defaults, closure, dct, base_globals) From a48ef1b002b7ca054a4c95c4e49b9588c3337cba Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Mon, 18 Mar 2019 11:03:47 +0100 Subject: [PATCH 5/6] MNT update changelog --- CHANGES.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 160e0d68f..9658a676b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,10 @@ +0.8.1 +===== + +- Fix a bug (already present before 0.5.3 and re-introduced in 0.8.0) + affecting relative import instructions inside depickled functions + ([issue #254](https://github.com/cloudpipe/cloudpickle/pull/254)) + 0.8.0 ===== From deb5bd947acd4555726adb59de14d286c1a75a29 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Wed, 20 Mar 2019 11:36:55 +0100 Subject: [PATCH 6/6] [ci downstream]