diff --git a/.travis.yml b/.travis.yml index 300244df6..7661144e2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,6 +12,9 @@ matrix: - os: linux dist: trusty python: "pypy3" + - os: linux + if: commit_message =~ /(\[ci python-nightly\])/ + env: PYTHON_NIGHTLY=1 - os: linux python: 3.7 - os: linux @@ -69,15 +72,30 @@ before_install: export PATH="$PYTHON_ROOT:$PYTHON_ROOT/Scripts:$PATH"; python -m pip install --upgrade pip; fi + - if [[ "$PYTHON_NIGHTLY" == 1 ]]; then + export VENV_DIR="$HOME/python38"; + pushd ..; + git clone https://github.com/python/cpython.git; + pushd cpython; + ./configure; + make; + ./python -m venv "$VENV_DIR"; + popd; + popd; + export PYTHON_EXE="$VENV_DIR/bin/python"; + else + export PYTHON_EXE="python"; + fi + install: - - pip install . - - pip install --upgrade -r dev-requirements.txt - - pip install tornado - - if [[ $TRAVIS_PYTHON_VERSION != 'pypy'* ]]; then - pip install numpy scipy; + - $PYTHON_EXE -m pip install . + - $PYTHON_EXE -m pip install --upgrade -r dev-requirements.txt + - $PYTHON_EXE -m pip install tornado + - if [[ $TRAVIS_PYTHON_VERSION != 'pypy'* && "$PYTHON_NIGHTLY" != 1 ]]; then + $PYTHON_EXE -m pip install numpy scipy; fi - if [[ $PROJECT != "" ]]; then - pip install $TEST_REQUIREMENTS; + $PYTHON_EXE -m pip install $TEST_REQUIREMENTS; pushd ..; git clone $PROJECT_URL; if [[ $PROJECT == "joblib" ]]; then @@ -85,22 +103,22 @@ install: source vendor_cloudpickle.sh ../../../cloudpickle; popd; fi; - pip install ./$PROJECT; + $PYTHON_EXE -m pip install ./$PROJECT; popd; fi - - pip list + - $PYTHON_EXE -m pip list before_script: # stop the build if there are Python syntax errors or undefined names - - flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics + - $PYTHON_EXE -m flake8 . --count --verbose --select=E901,E999,F821,F822,F823 --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - python ci/install_coverage_subprocess_pth.py + - $PYTHON_EXE -m flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - $PYTHON_EXE ci/install_coverage_subprocess_pth.py script: - - COVERAGE_PROCESS_START="$TRAVIS_BUILD_DIR/.coveragerc" PYTHONPATH='.:tests' pytest -r s + - COVERAGE_PROCESS_START="$TRAVIS_BUILD_DIR/.coveragerc" PYTHONPATH='.:tests' $PYTHON_EXE -m pytest -r s - | if [[ $PROJECT != "" ]]; then pushd ../$PROJECT - pytest -vl + $PYTHON_EXE -m pytest -vl TEST_RETURN_CODE=$? popd if [[ "$TEST_RETURN_CODE" != "0" ]]; then diff --git a/CHANGES.md b/CHANGES.md index e21ae720d..ee0101c67 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,9 @@ 1.1.0 ===== +- Support the pickling of interactively-defined functions with positional-only + arguments. ([issue #266](https://github.com/cloudpipe/cloudpickle/pull/266)) + - Track the provenance of dynamic classes and enums so as to preseve the usual `isinstance` relationship between pickled objects and their original class defintions. diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index d84cce76d..fcdac7510 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -168,24 +168,43 @@ def inner(value): (), ) else: - return types.CodeType( - co.co_argcount, - co.co_kwonlyargcount, - co.co_nlocals, - co.co_stacksize, - co.co_flags, - co.co_code, - co.co_consts, - co.co_names, - co.co_varnames, - co.co_filename, - co.co_name, - co.co_firstlineno, - co.co_lnotab, - co.co_cellvars, # this is the trickery - (), - ) - + if hasattr(types.CodeType, "co_posonlyargcount"): # pragma: no branch + return types.CodeType( + co.co_argcount, + co.co_posonlyargcount, # Python3.8 with PEP570 + co.co_kwonlyargcount, + co.co_nlocals, + co.co_stacksize, + co.co_flags, + co.co_code, + co.co_consts, + co.co_names, + co.co_varnames, + co.co_filename, + co.co_name, + co.co_firstlineno, + co.co_lnotab, + co.co_cellvars, # this is the trickery + (), + ) + else: + return types.CodeType( + co.co_argcount, + co.co_kwonlyargcount, + co.co_nlocals, + co.co_stacksize, + co.co_flags, + co.co_code, + co.co_consts, + co.co_names, + co.co_varnames, + co.co_filename, + co.co_name, + co.co_firstlineno, + co.co_lnotab, + co.co_cellvars, # this is the trickery + (), + ) _cell_set_template_code = _make_cell_set_template_code() @@ -371,12 +390,23 @@ def save_codeobject(self, obj): Save a code object """ if PY3: # pragma: no branch - args = ( - obj.co_argcount, obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, - obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, obj.co_varnames, - obj.co_filename, obj.co_name, obj.co_firstlineno, obj.co_lnotab, obj.co_freevars, - obj.co_cellvars - ) + if hasattr(obj, "co_posonlyargcount"): # pragma: no branch + args = ( + obj.co_argcount, obj.co_posonlyargcount, + obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, + obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, + obj.co_varnames, obj.co_filename, obj.co_name, + obj.co_firstlineno, obj.co_lnotab, obj.co_freevars, + obj.co_cellvars + ) + else: + args = ( + obj.co_argcount, obj.co_kwonlyargcount, obj.co_nlocals, + obj.co_stacksize, obj.co_flags, obj.co_code, obj.co_consts, + obj.co_names, obj.co_varnames, obj.co_filename, + obj.co_name, obj.co_firstlineno, obj.co_lnotab, + obj.co_freevars, obj.co_cellvars + ) else: args = ( obj.co_argcount, obj.co_nlocals, obj.co_stacksize, obj.co_flags, obj.co_code, diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py index 8f358ac64..4fb1e2e4e 100644 --- a/tests/cloudpickle_test.py +++ b/tests/cloudpickle_test.py @@ -1639,6 +1639,32 @@ def f(a, *, b=1): """.format(protocol=self.protocol) assert_run_python_script(textwrap.dedent(code)) + @pytest.mark.skipif(not hasattr(types.CodeType, "co_posonlyargcount"), + reason="Requires positional-only argument syntax") + def test_interactively_defined_func_with_positional_only_argument(self): + # Fixes https://github.com/cloudpipe/cloudpickle/issues/266 + # The source code of this test is bundled in a string and is ran from + # the __main__ module of a subprocess in order to avoid a SyntaxError + # in versions of python that do not support positional-only argument + # syntax. + code = """ + import pytest + from cloudpickle import loads, dumps + + def f(a, /, b=1): + return a + b + + depickled_f = loads(dumps(f, protocol={protocol})) + + for func in (f, depickled_f): + assert func(2) == 3 + assert func.__code__.co_posonlyargcount == 1 + with pytest.raises(TypeError): + func(a=2) + + """.format(protocol=self.protocol) + assert_run_python_script(textwrap.dedent(code)) + class Protocol2CloudPickleTest(CloudPickleTest): protocol = 2