diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index 654b8aacf..563f1af55 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -77,6 +77,13 @@ _DYNAMIC_CLASS_TRACKER_BY_ID = weakref.WeakValueDictionary() _DYNAMIC_CLASS_TRACKER_LOCK = threading.Lock() +PYPY = platform.python_implementation() == "PyPy" + +builtin_code_type = None +if PYPY: + # builtin-code objects only exist in pypy + builtin_code_type = type(float.__new__.__code__) + if sys.version_info[0] < 3: # pragma: no branch from pickle import Pickler try: @@ -453,12 +460,40 @@ def save_function(self, obj, name=None): Determines what kind of function obj is (e.g. lambda, defined at interactive prompt, etc) and handles the pickling appropriately. """ - if not _is_global(obj, name=name): + if _is_global(obj, name=name): + return Pickler.save_global(self, obj, name=name) + elif PYPY and isinstance(obj.__code__, builtin_code_type): + return self.save_pypy_builtin_func(obj) + else: return self.save_function_tuple(obj) - return Pickler.save_global(self, obj, name=name) dispatch[types.FunctionType] = save_function + def save_pypy_builtin_func(self, obj): + """Save pypy equivalent of builtin functions. + + PyPy does not have the concept of builtin-functions. Instead, + builtin-functions are simple function instances, but with a + builtin-code attribute. + Most of the time, builtin functions should be pickled by attribute. But + PyPy has flaky support for __qualname__, so some builtin functions such + as float.__new__ will be classified as dynamic. For this reason only, + we created this special routine. Because builtin-functions are not + expected to have closure or globals, there is no additional hack + (compared the one already implemented in pickle) to protect ourselves + from reference cycles. A simple (reconstructor, newargs, obj.__dict__) + tuple is save_reduced. + + Note also that PyPy improved their support for __qualname__ in v3.6, so + this routing should be removed when cloudpickle supports only PyPy 3.6 + and later. + """ + rv = (types.FunctionType, (obj.__code__, {}, obj.__name__, + obj.__defaults__, obj.__closure__), + obj.__dict__) + self.save_reduce(*rv, obj=obj) + + def _save_subimports(self, code, top_level_dependencies): """ Save submodules used by a function but not listed in its globals. @@ -676,10 +711,7 @@ def save_function_tuple(self, func): write(pickle.TUPLE) write(pickle.REDUCE) # applies _fill_function on the tuple - _extract_code_globals_cache = ( - weakref.WeakKeyDictionary() - if not hasattr(sys, "pypy_version_info") - else {}) + _extract_code_globals_cache = weakref.WeakKeyDictionary() @classmethod def extract_code_globals(cls, co): @@ -688,19 +720,14 @@ def extract_code_globals(cls, co): """ out_names = cls._extract_code_globals_cache.get(co) if out_names is None: - try: - names = co.co_names - except AttributeError: - # PyPy "builtin-code" object - out_names = set() - else: - out_names = {names[oparg] for _, oparg in _walk_global_ops(co)} - - # see if nested function have any global refs - if co.co_consts: - for const in co.co_consts: - if type(const) is types.CodeType: - out_names |= cls.extract_code_globals(const) + names = co.co_names + out_names = {names[oparg] for _, oparg in _walk_global_ops(co)} + + # see if nested function have any global refs + if co.co_consts: + for const in co.co_consts: + if isinstance(const, types.CodeType): + out_names |= cls.extract_code_globals(const) cls._extract_code_globals_cache[co] = out_names