Skip to content

Commit 87ee20c

Browse files
tqchentqchen
committed
[PYTHON] Build cython by default
Historically we have been maintaining cython and ctypes for FFI. One reason was cython was not that available then. Cython have quite good availability now and also enjoys great perf benefit. This PR changes the CMake to make cython as required dependency of the python package and build cython by default. It also removes ctypes testcases in favor of all cython tests. If things go well, we can consider remove ctypes and focus on cython support onward.
1 parent e5cea6d commit 87ee20c

15 files changed

+68
-68
lines changed

‎CMakeLists.txt‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -819,6 +819,18 @@ endif()
819819
# Custom targets
820820
add_custom_target(runtime DEPENDS tvm_runtime)
821821

822+
# By default add cython to all build
823+
find_package(Python)
824+
if(NOT DEFINED ENV{CONDA_BUILD})
825+
message(STATUS ${CMAKE_CURRENT_BINARY_DIR})
826+
add_custom_target(
827+
tvm_cython ALL
828+
${Python_EXECUTABLE} setup.py build_ext --inplace
829+
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/python
830+
)
831+
# add_dependencies(tvm_cython tvm)
832+
endif()
833+
822834
# Installation rules
823835
install(TARGETS tvm EXPORT ${PROJECT_NAME}Targets DESTINATION lib${LIB_SUFFIX})
824836
install(TARGETS tvm_runtime EXPORT ${PROJECT_NAME}Targets DESTINATION lib${LIB_SUFFIX})

‎Makefile‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@
2121
lint pylint cpplint \
2222
cppdoc docs \
2323
web webclean \
24-
cython cython3 cyclean \
2524
clean
2625

2726
.SECONDEXPANSION:

‎apps/benchmark/adreno/bench.sh‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,8 +48,6 @@ trap "{ kill ${TRACKER_PID}; kill ${DEVICE_PID}; }" 0
4848

4949
# cleanup pycache
5050
find . -type f -path "*.pyc" | xargs rm -f
51-
# Test TVM
52-
make cython3
5351

5452
if [ "texture" == $1 ] ; then
5553
python3 apps/benchmark/adreno/adreno_gpu_bench_texture.py --host ${TVM_TRACKER_HOST} --port ${TVM_TRACKER_PORT} --rpc-key ${RPC_DEVICE_KEY}

‎python/setup.py‎

Lines changed: 9 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@
2020
import pathlib
2121
import shutil
2222
import sys
23-
import sysconfig
2423

2524
from setuptools import find_packages
2625
from setuptools.dist import Distribution
@@ -38,7 +37,6 @@
3837
CONDA_BUILD = os.getenv("CONDA_BUILD") is not None
3938
INPLACE_BUILD = "--inplace" in sys.argv
4039

41-
4240
def get_lib_path():
4341
"""Get library path, name and version"""
4442
# We can not import `libinfo.py` in setup.py directly since __init__.py
@@ -133,24 +131,17 @@ def _remove_path(path):
133131

134132
def config_cython():
135133
"""Try to configure cython and return cython configuration"""
136-
if FFI_MODE not in ("cython"):
137-
if os.name == "nt" and not CONDA_BUILD:
138-
print("WARNING: Cython is not supported on Windows, will compile without cython module")
139-
return []
140-
sys_cflags = sysconfig.get_config_var("CFLAGS")
141-
if sys_cflags and "i386" in sys_cflags and "x86_64" in sys_cflags:
142-
print("WARNING: Cython library may not be compiled correctly with both i386 and x64")
143-
return []
134+
# Enforce cython unless FFI_MODE is explicitly set to ctypes
135+
# we might consider fully converge to cython later
136+
if FFI_MODE == "ctypes":
137+
return []
144138
try:
145139
from Cython.Build import cythonize
146140

147-
# from setuptools.extension import Extension
148-
if sys.version_info >= (3, 0):
149-
subdir = "_cy3"
150-
else:
151-
subdir = "_cy2"
141+
subdir = "_cy3"
142+
152143
ret = []
153-
path = "tvm/_ffi/_cython"
144+
cython_source = "tvm/_ffi/_cython"
154145
extra_compile_args = ["-std=c++17", "-DDMLC_USE_LOGGING_LIBRARY=<tvm/runtime/logging.h>"]
155146
if os.name == "nt":
156147
library_dirs = ["tvm", "../build/Release", "../build"]
@@ -166,7 +157,7 @@ def config_cython():
166157
library_dirs = None
167158
libraries = None
168159

169-
for fn in os.listdir(path):
160+
for fn in os.listdir(cython_source):
170161
if not fn.endswith(".pyx"):
171162
continue
172163
ret.append(
@@ -186,10 +177,7 @@ def config_cython():
186177
)
187178
return cythonize(ret, compiler_directives={"language_level": 3})
188179
except ImportError as error:
189-
if FFI_MODE == "cython":
190-
raise error
191-
print("WARNING: Cython is not installed, will compile without cython module")
192-
return []
180+
raise RuntimeError("Cython is not installed, please pip install cython")
193181

194182

195183
class BinaryDistribution(Distribution):

‎tests/scripts/setup-pytest-env.sh‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -47,15 +47,14 @@ function cleanup() {
4747
trap cleanup 0
4848

4949
function run_pytest() {
50+
ffi_type="cython"
5051
set -e
51-
local ffi_type="$1"
52-
shift
5352
local test_suite_name="$1"
5453
shift
5554
extra_args=( "$@" )
5655
if [ -z "${ffi_type}" -o -z "${test_suite_name}" ]; then
57-
echo "error: run_pytest called incorrectly: run_pytest ${ffi_type} ${test_suite_name}" "${extra_args[@]}"
58-
echo "usage: run_pytest <FFI_TYPE> <TEST_SUITE_NAME> [pytest args...]"
56+
echo "error: run_pytest called incorrectly: run_pytest ${test_suite_name}" "${extra_args[@]}"
57+
echo "usage: run_pytest <TEST_SUITE_NAME> [pytest args...]"
5958
exit 2
6059
fi
6160

@@ -85,7 +84,7 @@ function run_pytest() {
8584

8685
exit_code=0
8786
set +e
88-
TVM_FFI=${ffi_type} python3 -m pytest \
87+
python3 -m pytest \
8988
-o "junit_suite_name=${suite_name}" \
9089
"--junit-xml=${TVM_PYTEST_RESULT_DIR}/${suite_name}.xml" \
9190
"--junit-prefix=${ffi_type}" \

‎tests/scripts/task_python_adreno.sh‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -57,8 +57,8 @@ trap "{ kill ${TRACKER_PID}; kill ${DEVICE_PID}; cleanup; }" 0
5757

5858
# cleanup pycache
5959
find . -type f -path "*.pyc" | xargs rm -f
60-
# Test TVM
61-
make cython3
60+
# setup cython
61+
cd python; python3 setup.py build_ext --inplace; cd ..
6262

6363
# The RPC to remote Android device has issue of hang after few tests with in CI environments.
6464
# Lets run them individually on fresh rpc session.
@@ -68,7 +68,7 @@ i=0
6868
IFS=$'\n'
6969
for node_id in $TEXTURE_TESTS; do
7070
echo "$node_id"
71-
run_pytest ctypes "$TVM_INTEGRATION_TESTSUITE_NAME-opencl-texture-$i" "$node_id" --reruns=0
71+
run_pytest "$TVM_INTEGRATION_TESTSUITE_NAME-opencl-texture-$i" "$node_id" --reruns=0
7272
i=$((i+1))
7373
done
7474

@@ -77,7 +77,7 @@ CLML_TESTS=$(./ci/scripts/jenkins/pytest_ids.py --folder tests/python/contrib/te
7777
i=0
7878
for node_id in $CLML_TESTS; do
7979
echo "$node_id"
80-
CXX=${TVM_NDK_CC} run_pytest ctypes "$TVM_INTEGRATION_TESTSUITE_NAME-openclml-$i" "$node_id" --reruns=0
80+
CXX=${TVM_NDK_CC} run_pytest "$TVM_INTEGRATION_TESTSUITE_NAME-openclml-$i" "$node_id" --reruns=0
8181
i=$((i+1))
8282
done
8383

‎tests/scripts/task_python_arm_compute_library.sh‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,9 @@ set -euxo pipefail
2121
source tests/scripts/setup-pytest-env.sh
2222

2323

24-
# Rebuild cython
25-
# TODO(u99127): Enable cython tests.
26-
2724
find . -type f -path "*.pyc" | xargs rm -f
28-
make cython3
2925

30-
run_pytest ctypes python-arm_compute_lib tests/python/contrib/test_arm_compute_lib
26+
# setup cython
27+
cd python; python3 setup.py build_ext --inplace; cd ..
28+
29+
run_pytest python-arm_compute_lib tests/python/contrib/test_arm_compute_lib

‎tests/scripts/task_python_docs.sh‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,9 @@ clean_files() {
4646
sphinx_precheck() {
4747
clean_files
4848
echo "PreCheck sphinx doc generation WARNINGS.."
49-
make cython3
49+
50+
# setup cython
51+
cd python; python3 setup.py build_ext --inplace; cd ..
5052

5153
pushd docs
5254
make clean
@@ -123,7 +125,10 @@ clean_files
123125
# cleanup stale log files
124126
find . -type f -path "*.log" | xargs rm -f
125127
find . -type f -path "*.pyc" | xargs rm -f
126-
make cython3
128+
129+
# setup cython
130+
cd python; python3 setup.py build_ext --inplace; cd ..
131+
127132

128133
cd docs
129134
PYTHONPATH=$(pwd)/../python make htmldepoly SPHINXOPTS='-j auto' |& tee /tmp/$$.log.txt

‎tests/scripts/task_python_hexagon.sh‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,10 @@ if [ $# -ge 1 ] && [[ "$1" = "--device" ]]; then
2626
fi
2727

2828
source tests/scripts/setup-pytest-env.sh
29-
make cython3
29+
30+
# setup cython
31+
cd python; python3 setup.py build_ext --inplace; cd ..
32+
3033

3134
if [[ "${device_serial}" == "simulator" ]]; then
3235
export TVM_TRACKER_PORT=9190
@@ -49,9 +52,9 @@ fi
4952

5053
export ANDROID_SERIAL_NUMBER=${device_serial}
5154
if [ "${device_serial}" == "simulator" ]; then
52-
run_pytest ctypes python-contrib-hexagon tests/python/contrib/test_hexagon
55+
run_pytest python-contrib-hexagon tests/python/contrib/test_hexagon
5356
else
54-
run_pytest ctypes python-contrib-hexagon tests/python/contrib/test_hexagon -n=$num_of_devices
57+
run_pytest python-contrib-hexagon tests/python/contrib/test_hexagon -n=$num_of_devices
5558
fi
5659

5760
if [[ "${device_serial}" == "simulator" ]]; then

‎tests/scripts/task_python_integration.sh‎

Lines changed: 7 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -33,23 +33,20 @@ fi
3333
# cleanup pycache
3434
find . -type f -path "*.pyc" | xargs rm -f
3535

36-
# Test TVM
37-
make cython3
36+
# setup cython
37+
cd python; python3 setup.py build_ext --inplace; cd ..
3838

39-
run_pytest ctypes ${TVM_INTEGRATION_TESTSUITE_NAME}-integration tests/python/integration
39+
run_pytest ${TVM_INTEGRATION_TESTSUITE_NAME}-integration tests/python/integration
4040

4141
# forked is needed because the global registry gets contaminated
4242
TVM_TEST_TARGETS="${TVM_RELAY_TEST_TARGETS:-llvm;cuda}" \
43-
run_pytest ctypes ${TVM_INTEGRATION_TESTSUITE_NAME}-relay tests/python/relay --ignore=tests/python/relay/aot
43+
run_pytest ${TVM_INTEGRATION_TESTSUITE_NAME}-relay tests/python/relay --ignore=tests/python/relay/aot
4444

4545
# OpenCL texture test. Deselected specific tests that fails in CI
4646
TVM_TEST_TARGETS="${TVM_RELAY_OPENCL_TEXTURE_TARGETS:-opencl}" \
47-
run_pytest ctypes ${TVM_INTEGRATION_TESTSUITE_NAME}-opencl-texture tests/python/relay/opencl_texture
47+
run_pytest ${TVM_INTEGRATION_TESTSUITE_NAME}-opencl-texture tests/python/relay/opencl_texture
4848
# Command line driver test
49-
run_pytest ctypes ${TVM_INTEGRATION_TESTSUITE_NAME}-driver tests/python/driver
49+
run_pytest ${TVM_INTEGRATION_TESTSUITE_NAME}-driver tests/python/driver
5050

5151
# Target test
52-
run_pytest ctypes ${TVM_INTEGRATION_TESTSUITE_NAME}-target tests/python/target
53-
54-
# Do not enable OpenGL
55-
# run_pytest ctypes ${TVM_INTEGRATION_TESTSUITE_NAME}-webgl tests/webgl
52+
run_pytest ${TVM_INTEGRATION_TESTSUITE_NAME}-target tests/python/target

0 commit comments

Comments
 (0)