diff --git a/CMakeLists.txt b/CMakeLists.txt
index 757b0d1a89ff..f43eab264d20 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -819,6 +819,18 @@ endif()
 # Custom targets
 add_custom_target(runtime DEPENDS tvm_runtime)
 
+# By default add cython to all build
+find_package(Python)
+if(NOT DEFINED ENV{CONDA_BUILD})
+  message(STATUS ${CMAKE_CURRENT_BINARY_DIR})
+  add_custom_target(
+    tvm_cython ALL
+    ${Python_EXECUTABLE} setup.py build_ext --inplace
+    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/python
+  )
+  add_dependencies(tvm_cython tvm)
+endif()
+
 # Installation rules
 install(TARGETS tvm EXPORT ${PROJECT_NAME}Targets DESTINATION lib${LIB_SUFFIX})
 install(TARGETS tvm_runtime EXPORT ${PROJECT_NAME}Targets DESTINATION lib${LIB_SUFFIX})
diff --git a/Makefile b/Makefile
index 3a880b94a5af..5134c949ea48 100644
--- a/Makefile
+++ b/Makefile
@@ -21,7 +21,6 @@
         lint pylint cpplint \
 	cppdoc docs \
 	web webclean \
-	cython cython3 cyclean \
         clean
 
 .SECONDEXPANSION:
diff --git a/apps/benchmark/adreno/bench.sh b/apps/benchmark/adreno/bench.sh
index 7f9adeea5251..2c8ff288202f 100755
--- a/apps/benchmark/adreno/bench.sh
+++ b/apps/benchmark/adreno/bench.sh
@@ -48,8 +48,6 @@ trap "{ kill ${TRACKER_PID}; kill ${DEVICE_PID}; }" 0
 
 # cleanup pycache
 find . -type f -path "*.pyc" | xargs rm -f
-# Test TVM
-make cython3
 
 if [ "texture" == $1 ] ; then
     python3 apps/benchmark/adreno/adreno_gpu_bench_texture.py --host ${TVM_TRACKER_HOST} --port ${TVM_TRACKER_PORT} --rpc-key ${RPC_DEVICE_KEY}
diff --git a/python/setup.py b/python/setup.py
index 55f29c651af4..6cca9060bd2d 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -20,7 +20,6 @@
 import pathlib
 import shutil
 import sys
-import sysconfig
 
 from setuptools import find_packages
 from setuptools.dist import Distribution
@@ -133,24 +132,17 @@ def _remove_path(path):
 
 def config_cython():
     """Try to configure cython and return cython configuration"""
-    if FFI_MODE not in ("cython"):
-        if os.name == "nt" and not CONDA_BUILD:
-            print("WARNING: Cython is not supported on Windows, will compile without cython module")
-            return []
-        sys_cflags = sysconfig.get_config_var("CFLAGS")
-        if sys_cflags and "i386" in sys_cflags and "x86_64" in sys_cflags:
-            print("WARNING: Cython library may not be compiled correctly with both i386 and x64")
-            return []
+    # Enforce cython unless FFI_MODE is explicitly set to ctypes
+    # we might consider fully converge to cython later
+    if FFI_MODE == "ctypes":
+        return []
     try:
         from Cython.Build import cythonize
 
-        # from setuptools.extension import Extension
-        if sys.version_info >= (3, 0):
-            subdir = "_cy3"
-        else:
-            subdir = "_cy2"
+        subdir = "_cy3"
+
         ret = []
-        path = "tvm/_ffi/_cython"
+        cython_source = "tvm/_ffi/_cython"
         extra_compile_args = ["-std=c++17", "-DDMLC_USE_LOGGING_LIBRARY=<tvm/runtime/logging.h>"]
         if os.name == "nt":
             library_dirs = ["tvm", "../build/Release", "../build"]
@@ -166,7 +158,7 @@ def config_cython():
             library_dirs = None
             libraries = None
 
-        for fn in os.listdir(path):
+        for fn in os.listdir(cython_source):
             if not fn.endswith(".pyx"):
                 continue
             ret.append(
@@ -186,10 +178,7 @@ def config_cython():
             )
         return cythonize(ret, compiler_directives={"language_level": 3})
     except ImportError as error:
-        if FFI_MODE == "cython":
-            raise error
-        print("WARNING: Cython is not installed, will compile without cython module")
-        return []
+        raise RuntimeError("Cython is not installed, please pip install cython")
 
 
 class BinaryDistribution(Distribution):
diff --git a/tests/scripts/setup-pytest-env.sh b/tests/scripts/setup-pytest-env.sh
index 7bdfbdd1df47..171ddbc2d0d6 100755
--- a/tests/scripts/setup-pytest-env.sh
+++ b/tests/scripts/setup-pytest-env.sh
@@ -47,15 +47,14 @@ function cleanup() {
 trap cleanup 0
 
 function run_pytest() {
+    ffi_type="cython"
     set -e
-    local ffi_type="$1"
-    shift
     local test_suite_name="$1"
     shift
     extra_args=( "$@" )
     if [ -z "${ffi_type}" -o -z "${test_suite_name}" ]; then
-        echo "error: run_pytest called incorrectly: run_pytest ${ffi_type} ${test_suite_name}" "${extra_args[@]}"
-        echo "usage: run_pytest <FFI_TYPE> <TEST_SUITE_NAME> [pytest args...]"
+        echo "error: run_pytest called incorrectly: run_pytest ${test_suite_name}" "${extra_args[@]}"
+        echo "usage: run_pytest <TEST_SUITE_NAME> [pytest args...]"
         exit 2
     fi
 
@@ -85,7 +84,7 @@ function run_pytest() {
 
     exit_code=0
     set +e
-    TVM_FFI=${ffi_type} python3 -m pytest \
+    python3 -m pytest \
            -o "junit_suite_name=${suite_name}" \
            "--junit-xml=${TVM_PYTEST_RESULT_DIR}/${suite_name}.xml" \
            "--junit-prefix=${ffi_type}" \
diff --git a/tests/scripts/task_python_adreno.sh b/tests/scripts/task_python_adreno.sh
index e20d1bea31c5..a435d588aefb 100755
--- a/tests/scripts/task_python_adreno.sh
+++ b/tests/scripts/task_python_adreno.sh
@@ -57,8 +57,8 @@ trap "{ kill ${TRACKER_PID}; kill ${DEVICE_PID}; cleanup; }" 0
 
 # cleanup pycache
 find . -type f -path "*.pyc" | xargs rm -f
-# Test TVM
-make cython3
+# setup cython
+cd python; python3 setup.py build_ext --inplace; cd ..
 
 # The RPC to remote Android device has issue of hang after few tests with in CI environments.
 # Lets run them individually on fresh rpc session.
@@ -68,7 +68,7 @@ i=0
 IFS=$'\n'
 for node_id in $TEXTURE_TESTS; do
     echo "$node_id"
-    run_pytest ctypes "$TVM_INTEGRATION_TESTSUITE_NAME-opencl-texture-$i" "$node_id" --reruns=0
+    run_pytest "$TVM_INTEGRATION_TESTSUITE_NAME-opencl-texture-$i" "$node_id" --reruns=0
     i=$((i+1))
 done
 
@@ -77,7 +77,7 @@ CLML_TESTS=$(./ci/scripts/jenkins/pytest_ids.py --folder tests/python/contrib/te
 i=0
 for node_id in $CLML_TESTS; do
     echo "$node_id"
-    CXX=${TVM_NDK_CC} run_pytest ctypes "$TVM_INTEGRATION_TESTSUITE_NAME-openclml-$i" "$node_id" --reruns=0
+    CXX=${TVM_NDK_CC} run_pytest "$TVM_INTEGRATION_TESTSUITE_NAME-openclml-$i" "$node_id" --reruns=0
     i=$((i+1))
 done
 
diff --git a/tests/scripts/task_python_arm_compute_library.sh b/tests/scripts/task_python_arm_compute_library.sh
index 51b453e2b862..520abb58cdc2 100755
--- a/tests/scripts/task_python_arm_compute_library.sh
+++ b/tests/scripts/task_python_arm_compute_library.sh
@@ -21,10 +21,9 @@ set -euxo pipefail
 source tests/scripts/setup-pytest-env.sh
 
 
-# Rebuild cython
-# TODO(u99127): Enable cython tests.
-
 find . -type f -path "*.pyc" | xargs rm -f
-make cython3
 
-run_pytest ctypes python-arm_compute_lib tests/python/contrib/test_arm_compute_lib
+# setup cython
+cd python; python3 setup.py build_ext --inplace; cd ..
+
+run_pytest python-arm_compute_lib tests/python/contrib/test_arm_compute_lib
diff --git a/tests/scripts/task_python_docs.sh b/tests/scripts/task_python_docs.sh
index a72078d74011..b217f692b4a3 100755
--- a/tests/scripts/task_python_docs.sh
+++ b/tests/scripts/task_python_docs.sh
@@ -46,7 +46,9 @@ clean_files() {
 sphinx_precheck() {
     clean_files
     echo "PreCheck sphinx doc generation WARNINGS.."
-    make cython3
+
+    # setup cython
+    cd python; python3 setup.py build_ext --inplace; cd ..
 
     pushd docs
     make clean
@@ -123,7 +125,10 @@ clean_files
 # cleanup stale log files
 find . -type f -path "*.log" | xargs rm -f
 find . -type f -path "*.pyc" | xargs rm -f
-make cython3
+
+# setup cython
+cd python; python3 setup.py build_ext --inplace; cd ..
+
 
 cd docs
 PYTHONPATH=$(pwd)/../python make htmldepoly SPHINXOPTS='-j auto' |& tee /tmp/$$.log.txt
diff --git a/tests/scripts/task_python_hexagon.sh b/tests/scripts/task_python_hexagon.sh
index ba125b161a56..11e4e1827421 100755
--- a/tests/scripts/task_python_hexagon.sh
+++ b/tests/scripts/task_python_hexagon.sh
@@ -26,7 +26,10 @@ if [ $# -ge 1 ] && [[ "$1" = "--device" ]]; then
 fi
 
 source tests/scripts/setup-pytest-env.sh
-make cython3
+
+# setup cython
+cd python; python3 setup.py build_ext --inplace; cd ..
+
 
 if [[ "${device_serial}" == "simulator" ]]; then
     export TVM_TRACKER_PORT=9190
@@ -49,9 +52,9 @@ fi
 
 export ANDROID_SERIAL_NUMBER=${device_serial}
 if [ "${device_serial}" == "simulator" ]; then
-    run_pytest ctypes python-contrib-hexagon tests/python/contrib/test_hexagon
+    run_pytest python-contrib-hexagon tests/python/contrib/test_hexagon
 else
-    run_pytest ctypes python-contrib-hexagon tests/python/contrib/test_hexagon -n=$num_of_devices
+    run_pytest python-contrib-hexagon tests/python/contrib/test_hexagon -n=$num_of_devices
 fi
 
 if [[ "${device_serial}" == "simulator" ]]; then
diff --git a/tests/scripts/task_python_integration.sh b/tests/scripts/task_python_integration.sh
index da0d0500837e..51ef86d05ec7 100755
--- a/tests/scripts/task_python_integration.sh
+++ b/tests/scripts/task_python_integration.sh
@@ -33,23 +33,20 @@ fi
 # cleanup pycache
 find . -type f -path "*.pyc" | xargs rm -f
 
-# Test TVM
-make cython3
+# setup cython
+cd python; python3 setup.py build_ext --inplace; cd ..
 
-run_pytest ctypes ${TVM_INTEGRATION_TESTSUITE_NAME}-integration tests/python/integration
+run_pytest ${TVM_INTEGRATION_TESTSUITE_NAME}-integration tests/python/integration
 
 # forked is needed because the global registry gets contaminated
 TVM_TEST_TARGETS="${TVM_RELAY_TEST_TARGETS:-llvm;cuda}" \
-    run_pytest ctypes ${TVM_INTEGRATION_TESTSUITE_NAME}-relay tests/python/relay --ignore=tests/python/relay/aot
+    run_pytest ${TVM_INTEGRATION_TESTSUITE_NAME}-relay tests/python/relay --ignore=tests/python/relay/aot
 
 # OpenCL texture test. Deselected specific tests that fails  in CI
 TVM_TEST_TARGETS="${TVM_RELAY_OPENCL_TEXTURE_TARGETS:-opencl}" \
-    run_pytest ctypes ${TVM_INTEGRATION_TESTSUITE_NAME}-opencl-texture tests/python/relay/opencl_texture
+    run_pytest ${TVM_INTEGRATION_TESTSUITE_NAME}-opencl-texture tests/python/relay/opencl_texture
 # Command line driver test
-run_pytest ctypes ${TVM_INTEGRATION_TESTSUITE_NAME}-driver tests/python/driver
+run_pytest ${TVM_INTEGRATION_TESTSUITE_NAME}-driver tests/python/driver
 
 # Target test
-run_pytest ctypes ${TVM_INTEGRATION_TESTSUITE_NAME}-target tests/python/target
-
-# Do not enable OpenGL
-# run_pytest ctypes ${TVM_INTEGRATION_TESTSUITE_NAME}-webgl tests/webgl
+run_pytest ${TVM_INTEGRATION_TESTSUITE_NAME}-target tests/python/target
diff --git a/tests/scripts/task_python_nightly.sh b/tests/scripts/task_python_nightly.sh
index f8423602bd4b..42cf343e71ad 100755
--- a/tests/scripts/task_python_nightly.sh
+++ b/tests/scripts/task_python_nightly.sh
@@ -20,10 +20,10 @@ set -euxo pipefail
 
 source tests/scripts/setup-pytest-env.sh
 
-# Rebuild cython
-make cython3
+# setup cython
+cd python; python3 setup.py build_ext --inplace; cd ..
 
 # cleanup pycache
 find . -type f -path "*.pyc" | xargs rm -f
 
-run_pytest cython python-topi-nightly tests/python/topi/nightly
+run_pytest python-topi-nightly tests/python/topi/nightly
diff --git a/tests/scripts/task_python_topi.sh b/tests/scripts/task_python_topi.sh
index 3916e0a754f3..97030d6948f9 100755
--- a/tests/scripts/task_python_topi.sh
+++ b/tests/scripts/task_python_topi.sh
@@ -26,10 +26,10 @@ export TVM_TEST_TARGETS="llvm; cuda"
 export TVM_BIND_THREADS=0
 export OMP_NUM_THREADS=1
 
-# Rebuild cython
-make cython3
+# setup cython
+cd python; python3 setup.py build_ext --inplace; cd ..
 
 # cleanup pycache
 find . -type f -path "*.pyc" | xargs rm -f
 
-run_pytest cython python-topi tests/python/topi/
+run_pytest python-topi tests/python/topi/
diff --git a/tests/scripts/task_python_unittest.sh b/tests/scripts/task_python_unittest.sh
index 4a13c6ce1ed2..b84694cfd5e2 100755
--- a/tests/scripts/task_python_unittest.sh
+++ b/tests/scripts/task_python_unittest.sh
@@ -22,7 +22,9 @@ source tests/scripts/setup-pytest-env.sh
 
 # cleanup pycache
 find . -type f -path "*.pyc" | xargs rm -f
-make cython3
+
+# setup cython
+cd python; python3 setup.py build_ext --inplace; cd ..
 
 # NOTE: also set by task_python_unittest_gpuonly.sh.
 if [ -z "${TVM_UNITTEST_TESTSUITE_NAME:-}" ]; then
@@ -30,8 +32,7 @@ if [ -z "${TVM_UNITTEST_TESTSUITE_NAME:-}" ]; then
 fi
 
 # First run minimal test on both ctypes and cython.
-run_pytest ctypes ${TVM_UNITTEST_TESTSUITE_NAME}-platform-minimal-test-0 tests/python/all-platform-minimal-test
-run_pytest cython ${TVM_UNITTEST_TESTSUITE_NAME}-platform-minimal-test-1 tests/python/all-platform-minimal-test
+run_pytest ${TVM_UNITTEST_TESTSUITE_NAME}-platform-minimal-test tests/python/all-platform-minimal-test
 
 # Then run all unittests on both ctypes and cython.
 TEST_FILES=(
@@ -52,11 +53,9 @@ TEST_FILES=(
   "tir-usmp"
   "tvmscript"
   "usmp"
+  "ci"
 )
 
 for TEST_FILE in ${TEST_FILES[@]}; do
-    run_pytest cython ${TEST_FILE}-1, tests/python/${TEST_FILE}
+    run_pytest ${TEST_FILE}, tests/python/${TEST_FILE}
 done
-
-# Then run CI tests
-run_pytest ctypes ${TVM_UNITTEST_TESTSUITE_NAME}-ci tests/python/ci
diff --git a/tests/scripts/task_python_unittest_gpuonly.sh b/tests/scripts/task_python_unittest_gpuonly.sh
index e68fcba25c91..6c4e642b0c6b 100755
--- a/tests/scripts/task_python_unittest_gpuonly.sh
+++ b/tests/scripts/task_python_unittest_gpuonly.sh
@@ -33,4 +33,4 @@ export TVM_UNITTEST_TESTSUITE_NAME=python-codegen-vulkan
 
 source tests/scripts/setup-pytest-env.sh
 
-run_pytest cython ${TVM_UNITTEST_TESTSUITE_NAME}-1 tests/python/codegen/test_target_codegen_vulkan.py
+run_pytest ${TVM_UNITTEST_TESTSUITE_NAME}-1 tests/python/codegen/test_target_codegen_vulkan.py
diff --git a/tests/scripts/unity/task_python_relax.sh b/tests/scripts/unity/task_python_relax.sh
index 86d917c655bb..28dd78bf6bd2 100755
--- a/tests/scripts/unity/task_python_relax.sh
+++ b/tests/scripts/unity/task_python_relax.sh
@@ -25,7 +25,8 @@ export LD_LIBRARY_PATH="build:${LD_LIBRARY_PATH:-}"
 export TVM_BIND_THREADS=0
 export TVM_NUM_THREADS=2
 
-make cython3
+# setup cython
+cd python; python3 setup.py build_ext --inplace; cd ..
 
 # Run Relax tests
 TVM_TEST_TARGETS="${TVM_RELAY_TEST_TARGETS:-llvm}" pytest tests/python/relax