Skip to content
This repository has been archived by the owner on Nov 17, 2023. It is now read-only.

Python 2 cleanup #17583

Merged
merged 5 commits into from
Feb 15, 2020
Merged

Python 2 cleanup #17583

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
c40bffe
Drop _cy2
leezu Feb 13, 2020
91c5938
Drop Python 2 specific code in mxnet and tests
leezu Feb 13, 2020
e0e7e52
Replace io.open with open
leezu Feb 13, 2020
ba8fe07
Drop from __future__ imports
leezu Feb 13, 2020
ef59f58
Fix lint
leezu Feb 13, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -897,7 +897,7 @@ add_custom_target(mxnet_lint COMMAND ${CMAKE_COMMAND} -DMSVC=${MSVC} -DPYTHON_EX
if(BUILD_CYTHON_MODULES)
include(cmake/BuildCythonModules.cmake)
add_cython_modules(3) # Build cython module for python3 if python3 is found
if((NOT ${PYTHON2_FOUND}) AND (NOT ${PYTHON3_FOUND}))
if(NOT ${PYTHON3_FOUND})
message(FATAL_ERROR "No python interpreter found to build cython modules")
endif()
endif()
50 changes: 15 additions & 35 deletions amalgamation/python/mxnet_predict.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,46 +55,26 @@

__all__ = ["Predictor", "load_ndarray_file"]

if sys.version_info[0] == 3:
py_str = lambda x: x.decode('utf-8')

def c_str_array(strings):
"""Create ctypes const char ** from a list of Python strings.
py_str = lambda x: x.decode('utf-8')

Parameters
----------
strings : list of string
Python strings.

Returns
-------
(ctypes.c_char_p * len(strings))
A const char ** pointer that can be passed to C API.
"""
arr = (ctypes.c_char_p * len(strings))()
arr[:] = [s.encode('utf-8') for s in strings]
return arr
def c_str_array(strings):
"""Create ctypes const char ** from a list of Python strings.
Parameters
----------
strings : list of string
Python strings.
else:
py_str = lambda x: x

def c_str_array(strings):
"""Create ctypes const char ** from a list of Python strings.
Parameters
----------
strings : list of strings
Python strings.
Returns
-------
(ctypes.c_char_p * len(strings))
A const char ** pointer that can be passed to C API.
"""
arr = (ctypes.c_char_p * len(strings))()
arr[:] = strings
return arr
Returns
-------
(ctypes.c_char_p * len(strings))
A const char ** pointer that can be passed to C API.
"""
arr = (ctypes.c_char_p * len(strings))()
arr[:] = [s.encode('utf-8') for s in strings]
return arr


def c_str(string):
Expand Down
7 changes: 3 additions & 4 deletions ci/docker/runtime_functions.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,11 +43,10 @@ scala_prepare() {

check_cython() {
set -ex
local python_ver=$1
local is_cython_used=$(python${python_ver} <<EOF
local is_cython_used=$(python3 <<EOF
leezu marked this conversation as resolved.
Show resolved Hide resolved
import sys
import mxnet as mx
cython_ndarraybase = 'mxnet._cy' + str(sys.version_info.major) + '.ndarray'
cython_ndarraybase = 'mxnet._cy3.ndarray'
print(mx.nd._internal.NDArrayBase.__module__ == cython_ndarraybase)
EOF
)
Expand Down Expand Up @@ -1084,7 +1083,7 @@ unittest_ubuntu_python3_gpu_cython() {
export MXNET_ENABLE_CYTHON=1
export MXNET_ENFORCE_CYTHON=1
export DMLC_LOG_STACK_TRACE_DEPTH=10
check_cython 3
check_cython
nosetests-3.4 $NOSE_COVERAGE_ARGUMENTS $NOSE_TIMER_ARGUMENTS --with-xunit --xunit-file nosetests_gpu.xml --verbose tests/python/gpu
}

Expand Down
10 changes: 5 additions & 5 deletions ci/jenkins/Jenkins_steps.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,23 +24,23 @@ utils = load('ci/Jenkinsfile_utils.groovy')

// mxnet libraries
mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, lib/libtvm_runtime.so, lib/libtvmop.so, lib/tvmop.conf, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a'
mx_lib_cython = 'lib/libmxnet.so, lib/libmxnet.a, lib/libtvm_runtime.so, lib/libtvmop.so, lib/tvmop.conf, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, python/mxnet/_cy2/*.so, python/mxnet/_cy3/*.so'
mx_lib_cython = 'lib/libmxnet.so, lib/libmxnet.a, lib/libtvm_runtime.so, lib/libtvmop.so, lib/tvmop.conf, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, python/mxnet/_cy3/*.so'

// Python wheels
mx_pip = 'build/*.whl'

// mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default.
mx_cmake_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so'
mx_cmake_lib_no_tvm_op = 'build/libmxnet.so, build/libmxnet.a, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so'
mx_cmake_lib_cython = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, python/mxnet/_cy2/*.so, python/mxnet/_cy3/*.so'
mx_cmake_lib_cython = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, python/mxnet/_cy3/*.so'
// mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default.
mx_cmake_lib_debug = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests'
mx_cmake_mkldnn_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so'
mx_mkldnn_lib = 'lib/libmxnet.so, lib/libmxnet.a, lib/libtvm_runtime.so, lib/libtvmop.so, lib/tvmop.conf, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a'
mx_tensorrt_lib = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, lib/libnvonnxparser_runtime.so.0, lib/libnvonnxparser.so.0, lib/libonnx_proto.so, lib/libonnx.so'
mx_lib_cpp_examples = 'lib/libmxnet.so, lib/libmxnet.a, lib/libtvm_runtime.so, lib/libtvmop.so, lib/tvmop.conf, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, build/cpp-package/example/*, python/mxnet/_cy2/*.so, python/mxnet/_cy3/*.so'
mx_lib_cpp_capi = 'lib/libmxnet.so, lib/libmxnet.a, lib/libtvm_runtime.so, lib/libtvmop.so, lib/tvmop.conf, libsample_lib.so, lib/libmkldnn.so.1, lib/libmklml_intel.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, build/cpp-package/example/*, python/mxnet/_cy2/*.so, python/mxnet/_cy3/*.so, build/tests/cpp/mxnet_unit_tests'
mx_lib_cpp_examples_no_tvm_op = 'lib/libmxnet.so, lib/libmxnet.a, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, build/cpp-package/example/*, python/mxnet/_cy2/*.so, python/mxnet/_cy3/*.so'
mx_lib_cpp_examples = 'lib/libmxnet.so, lib/libmxnet.a, lib/libtvm_runtime.so, lib/libtvmop.so, lib/tvmop.conf, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, build/cpp-package/example/*, python/mxnet/_cy3/*.so'
mx_lib_cpp_capi = 'lib/libmxnet.so, lib/libmxnet.a, lib/libtvm_runtime.so, lib/libtvmop.so, lib/tvmop.conf, libsample_lib.so, lib/libmkldnn.so.1, lib/libmklml_intel.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, build/cpp-package/example/*, python/mxnet/_cy3/*.so, build/tests/cpp/mxnet_unit_tests'
mx_lib_cpp_examples_no_tvm_op = 'lib/libmxnet.so, lib/libmxnet.a, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, build/cpp-package/example/*, python/mxnet/_cy3/*.so'
mx_lib_cpp_examples_cpu = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/cpp-package/example/*'

// Python unittest for CPU
Expand Down
7 changes: 3 additions & 4 deletions docs/python_docs/python/tutorials/packages/gluon/text/gnmt.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@ Load MXNET and Gluon
import time
import random
import os
import io
import logging
import numpy as np
import mxnet as mx
Expand Down Expand Up @@ -226,11 +225,11 @@ EOS tokens to target sentence.
data_val_lengths = get_data_lengths(data_val)
data_test_lengths = get_data_lengths(data_test)
with io.open(os.path.join(save_dir, 'val_gt.txt'), 'w', encoding='utf-8') as of:
with open(os.path.join(save_dir, 'val_gt.txt'), 'w', encoding='utf-8') as of:
for ele in val_tgt_sentences:
of.write(' '.join(ele) + '\n')
with io.open(os.path.join(save_dir, 'test_gt.txt'), 'w', encoding='utf-8') as of:
with open(os.path.join(save_dir, 'test_gt.txt'), 'w', encoding='utf-8') as of:
for ele in test_tgt_sentences:
of.write(' '.join(ele) + '\n')
Expand Down Expand Up @@ -391,7 +390,7 @@ testing datasets.
def write_sentences(sentences, file_path):
with io.open(file_path, 'w', encoding='utf-8') as of:
with open(file_path, 'w', encoding='utf-8') as of:
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Isn't utf8 by default?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No. Default is platform dependent. https://docs.python.org/3/library/functions.html#open

for sent in sentences:
of.write(' '.join(sent) + '\n')
Expand Down
1 change: 0 additions & 1 deletion python/mxnet/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@

# coding: utf-8
"""MXNet: a concise, fast and flexible framework for deep learning."""
from __future__ import absolute_import

from .context import Context, current_context, cpu, gpu, cpu_pinned
from . import engine, error
Expand Down
1 change: 0 additions & 1 deletion python/mxnet/_ctypes/ndarray.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
# pylint: disable=invalid-name, protected-access, too-many-arguments
# pylint: disable=global-statement, unused-import
"""NDArray configuration API."""
from __future__ import absolute_import as _abs

import ctypes

Expand Down
1 change: 0 additions & 1 deletion python/mxnet/_ctypes/symbol.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
# coding: utf-8
# pylint: disable=invalid-name, protected-access, too-many-arguments, global-statement
"""Symbolic configuration API."""
from __future__ import absolute_import as _abs

import ctypes
from ..base import _LIB
Expand Down
1 change: 0 additions & 1 deletion python/mxnet/_cy2/README
View file
Open in desktop

This file was deleted.

18 changes: 0 additions & 18 deletions python/mxnet/_cy2/__init__.py
View file
Open in desktop

This file was deleted.

1 change: 0 additions & 1 deletion python/mxnet/attribute.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@

# coding: utf-8
"""Attribute scoping support for symbolic API."""
from __future__ import absolute_import
import threading
import warnings
from collections import defaultdict
Expand Down
2 changes: 0 additions & 2 deletions python/mxnet/autograd.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,6 @@

# coding: utf-8
"""Autograd for NDArray."""
from __future__ import absolute_import
from __future__ import division

from array import array
from threading import Lock
Expand Down
137 changes: 35 additions & 102 deletions python/mxnet/base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,7 @@
# coding: utf-8
# pylint: disable=invalid-name, no-member, trailing-comma-tuple, bad-mcs-classmethod-argument, unnecessary-pass, too-many-lines, wrong-import-position
"""ctypes library of mxnet and helper functions."""
from __future__ import absolute_import

import io
import re
import atexit
import ctypes
Expand Down Expand Up @@ -51,12 +49,9 @@
string_types = basestring,
error_types = {}

if sys.version_info[0] > 2:
# this function is needed for python3
# to convert ctypes.char_p .value back to python str
py_str = lambda x: x.decode('utf-8')
else:
py_str = lambda x: x
# this function is needed for python3
# to convert ctypes.char_p .value back to python str
py_str = lambda x: x.decode('utf-8')


def data_dir_default():
Expand All @@ -78,28 +73,6 @@ def data_dir():
"""
return os.getenv('MXNET_HOME', data_dir_default())

class _Py2CompatibleUnicodeFileWriter(object):
"""
Wraps a file handle decorating the write command to unicode the content before writing.
This makes writing files opened with encoding='utf-8' compatible with Python 2
"""

def __init__(self, file_handle):
self._file_handle = file_handle
if sys.version_info[0] > 2:
self.unicode = str
else:
from functools import partial
# pylint: disable=undefined-variable
self.unicode = partial(unicode, encoding="utf-8")
# pylint: enable=undefined-variable

def write(self, value):
self._file_handle.write(self.unicode(value))

def __getattr__(self, name):
return getattr(self._file_handle, name)


class _NullType(object):
"""Placeholder for arguments"""
Expand Down Expand Up @@ -404,83 +377,43 @@ def _load_lib():
#----------------------------
# helper function definition
#----------------------------
if sys.version_info[0] < 3:
def c_str(string):
"""Create ctypes char * from a Python string.
Parameters
----------
string : string type
Python string.
Returns
-------
str : c_char_p
A char pointer that can be passed to C API.
Examples
--------
>>> x = mx.base.c_str("Hello, World")
>>> print x.value
Hello, World
"""
return ctypes.c_char_p(string)
def c_str(string):
"""Create ctypes char * from a Python string.
def c_str_array(strings):
"""Create ctypes const char ** from a list of Python strings.
Parameters
----------
string : string type
Python string.
Parameters
----------
strings : list of string
Python strings.
Returns
-------
str : c_char_p
A char pointer that can be passed to C API.
Returns
-------
(ctypes.c_char_p * len(strings))
A const char ** pointer that can be passed to C API.
"""
arr = (ctypes.c_char_p * len(strings))()
arr[:] = strings
return arr

else:
def c_str(string):
"""Create ctypes char * from a Python string.
Parameters
----------
string : string type
Python string.
Returns
-------
str : c_char_p
A char pointer that can be passed to C API.
Examples
--------
>>> x = mx.base.c_str("Hello, World")
>>> print(x.value)
b"Hello, World"
"""
return ctypes.c_char_p(string.encode('utf-8'))
Examples
--------
>>> x = mx.base.c_str("Hello, World")
>>> print(x.value)
b"Hello, World"
"""
return ctypes.c_char_p(string.encode('utf-8'))

def c_str_array(strings):
"""Create ctypes const char ** from a list of Python strings.
def c_str_array(strings):
"""Create ctypes const char ** from a list of Python strings.
Parameters
----------
strings : list of string
Python strings.
Parameters
----------
strings : list of string
Python strings.
Returns
-------
(ctypes.c_char_p * len(strings))
A const char ** pointer that can be passed to C API.
"""
arr = (ctypes.c_char_p * len(strings))()
arr[:] = [s.encode('utf-8') for s in strings]
return arr
Returns
-------
(ctypes.c_char_p * len(strings))
A const char ** pointer that can be passed to C API.
"""
arr = (ctypes.c_char_p * len(strings))()
arr[:] = [s.encode('utf-8') for s in strings]
return arr


def c_array(ctype, values):
Expand Down Expand Up @@ -821,7 +754,7 @@ def get_module_file(module_name):
module_path = module_name.split('.')
module_path[-1] = 'gen_' + module_path[-1]
file_name = os.path.join(path, '..', *module_path) + '.py'
module_file = _Py2CompatibleUnicodeFileWriter(io.open(file_name, 'w', encoding="utf-8"))
module_file = open(file_name, 'w', encoding="utf-8")
dependencies = {'symbol': ['from ._internal import SymbolBase',
'from ..base import _Null'],
'ndarray': ['from ._internal import NDArrayBase',
Expand Down
Loading