From 67bc9d1c37a262328a621f0d372f961e608f59e2 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Wed, 31 May 2017 11:01:35 +0200 Subject: [PATCH 1/4] Fix #94: allow pickling Logger objects --- cloudpickle/cloudpickle.py | 6 +++++ tests/cloudpickle_test.py | 54 +++++++++++++++++++++++++------------- 2 files changed, 42 insertions(+), 18 deletions(-) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index 030d44a3f..ab396f6fa 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -47,6 +47,7 @@ import imp import io import itertools +import logging import opcode import operator import pickle @@ -795,6 +796,11 @@ def inject_addons(self): """Plug in system. Register additional pickling functions if modules already loaded""" pass + def save_logger(self, obj): + self.save_reduce(logging.getLogger, (obj.name,)) + + dispatch[logging.Logger] = save_logger + # Tornado support diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py index 19f1faf1f..46f4150e8 100644 --- a/tests/cloudpickle_test.py +++ b/tests/cloudpickle_test.py @@ -1,16 +1,26 @@ from __future__ import division -import imp -import unittest -import pytest -import pickle -import sys -import random + +import base64 import functools +import imp +from io import BytesIO import itertools +import logging +from operator import itemgetter, attrgetter +import pickle import platform -import textwrap -import base64 +import random import subprocess +import sys +import textwrap +import unittest + +try: + from StringIO import StringIO +except ImportError: + from io import StringIO + +import pytest try: # try importing numpy and scipy. These are not hard dependencies and @@ -27,16 +37,6 @@ except ImportError: tornado = None - -from operator import itemgetter, attrgetter - -try: - from StringIO import StringIO -except ImportError: - from io import StringIO - -from io import BytesIO - import cloudpickle from cloudpickle.cloudpickle import _find_module, _make_empty_cell, cell_set @@ -507,6 +507,24 @@ def test_cell_manipulation(self): msg='cell contents not set correctly', ) + def test_logger(self): + logger = logging.getLogger('cloudpickle.dummy_test_logger') + self.assertIs(pickle_depickle(logger), logger) + + dumped = cloudpickle.dumps(logger) + + code = """if 1: + import cloudpickle, logging + + logging.basicConfig(level=logging.INFO) + logger = cloudpickle.loads(%(dumped)r) + logger.info('hello') + """ % locals() + out = subprocess.check_output([sys.executable, "-c", code], + stderr=subprocess.STDOUT) + self.assertEqual(out.strip().decode(), + 'INFO:cloudpickle.dummy_test_logger:hello') + if __name__ == '__main__': unittest.main() From a1a0e6ca8dc201ada0de630bbdf7008fdc0ddd4e Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Wed, 31 May 2017 11:04:53 +0200 Subject: [PATCH 2/4] Pass `obj` to save_reduce --- cloudpickle/cloudpickle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index ab396f6fa..781d2c590 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -797,7 +797,7 @@ def inject_addons(self): pass def save_logger(self, obj): - self.save_reduce(logging.getLogger, (obj.name,)) + self.save_reduce(logging.getLogger, (obj.name,), obj=obj) dispatch[logging.Logger] = save_logger From ed8d7e613c5fd232e4591981e32634b63de9c416 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Wed, 31 May 2017 20:18:06 +0200 Subject: [PATCH 3/4] Try to fix test on 2.6 --- tests/cloudpickle_test.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py index 46f4150e8..ec928cee3 100644 --- a/tests/cloudpickle_test.py +++ b/tests/cloudpickle_test.py @@ -509,7 +509,7 @@ def test_cell_manipulation(self): def test_logger(self): logger = logging.getLogger('cloudpickle.dummy_test_logger') - self.assertIs(pickle_depickle(logger), logger) + self.assertTrue(pickle_depickle(logger) is logger) dumped = cloudpickle.dumps(logger) @@ -520,8 +520,11 @@ def test_logger(self): logger = cloudpickle.loads(%(dumped)r) logger.info('hello') """ % locals() - out = subprocess.check_output([sys.executable, "-c", code], - stderr=subprocess.STDOUT) + proc = subprocess.Popen([sys.executable, "-c", code], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + out, _ = proc.communicate() + self.assertEqual(proc.wait(), 0) self.assertEqual(out.strip().decode(), 'INFO:cloudpickle.dummy_test_logger:hello') From 26f91d7db102716b5cf2b3054b89164f1562ae75 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Wed, 31 May 2017 20:30:18 +0200 Subject: [PATCH 4/4] Fix test on 2.6 (finally) --- cloudpickle/cloudpickle.py | 6 ++++++ tests/cloudpickle_test.py | 3 ++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index 781d2c590..4dd3da055 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -570,6 +570,12 @@ def save_inst(self, obj): Supports __transient__""" cls = obj.__class__ + # Try the dispatch table (pickle module doesn't do it) + f = self.dispatch.get(cls) + if f: + f(self, obj) # Call unbound method with explicit self + return + memo = self.memo write = self.write save = self.save diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py index ec928cee3..6e1a69318 100644 --- a/tests/cloudpickle_test.py +++ b/tests/cloudpickle_test.py @@ -509,7 +509,8 @@ def test_cell_manipulation(self): def test_logger(self): logger = logging.getLogger('cloudpickle.dummy_test_logger') - self.assertTrue(pickle_depickle(logger) is logger) + pickled = pickle_depickle(logger) + self.assertTrue(pickled is logger, (pickled, logger)) dumped = cloudpickle.dumps(logger)