diff --git a/tests/micro/common/test_autotune.py b/tests/micro/common/test_autotune.py index 46f6d8889a9a..14c0290c7b4f 100644 --- a/tests/micro/common/test_autotune.py +++ b/tests/micro/common/test_autotune.py @@ -31,6 +31,9 @@ @pytest.mark.requires_hardware @tvm.testing.requires_micro +@pytest.mark.skip_boards( + ["nucleo_l4r5zi", "", "nucleo_f746zg", "stm32f746g_disco", "nrf5340dk_nrf5340_cpuapp"] +) def test_kws_autotune_workflow(platform, board, tmp_path): mod, params = fetch_model_from_url( url="https://github.com/tensorflow/tflite-micro/raw/main/tensorflow/lite/micro/examples/micro_speech/micro_speech.tflite", diff --git a/tests/micro/common/test_tvmc.py b/tests/micro/common/test_tvmc.py index 3aa7fec2f299..1e1249b65ed8 100644 --- a/tests/micro/common/test_tvmc.py +++ b/tests/micro/common/test_tvmc.py @@ -128,6 +128,9 @@ def test_tvmc_model_build_only(platform, board, output_dir): "output_dir,", [pathlib.Path("./tvmc_relative_path_test"), pathlib.Path(tempfile.mkdtemp())], ) +@pytest.mark.skip_boards( + ["nucleo_l4r5zi", "", "nucleo_f746zg", "stm32f746g_disco", "nrf5340dk_nrf5340_cpuapp"] +) def test_tvmc_model_run(platform, board, output_dir): target = tvm.micro.testing.get_target(platform, board) diff --git a/tests/micro/zephyr/test_ms_tuning.py b/tests/micro/zephyr/test_ms_tuning.py index 16d48ca4cdd6..560f5e09596a 100644 --- a/tests/micro/zephyr/test_ms_tuning.py +++ b/tests/micro/zephyr/test_ms_tuning.py @@ -22,6 +22,7 @@ import tvm from tvm import relay +import tvm.micro.testing from tvm.relay.backend import Executor from tvm.contrib import graph_executor, utils from tvm import meta_schedule as ms @@ -61,7 +62,7 @@ def create_relay_module(): @tvm.testing.requires_micro -@pytest.mark.skip_boards(["mps2_an521", "mps3_an547"]) +@pytest.mark.skip_boards(["mps2_an521", "mps3_an547", "nucleo_f746zg", "stm32f746g_disco"]) def test_ms_tuning_conv2d(workspace_dir, board, microtvm_debug, use_fvp, serial_number): """Test meta-schedule tuning for microTVM Zephyr""" @@ -92,7 +93,7 @@ def test_ms_tuning_conv2d(workspace_dir, board, microtvm_debug, use_fvp, serial_ boards_file = pathlib.Path(tvm.micro.get_microtvm_template_projects("zephyr")) / "boards.json" with open(boards_file) as f: boards = json.load(f) - target = tvm.target.target.micro(model=boards[project_options["board"]]["model"]) + target = tvm.micro.testing.get_target("zephyr", board) runtime = relay.backend.Runtime("crt", {"system-lib": True}) executor = Executor("aot", {"link-params": True}) @@ -158,7 +159,7 @@ def test_ms_tuning_conv2d(workspace_dir, board, microtvm_debug, use_fvp, serial_ # Build reference model (without tuning) dev = tvm.cpu() - target = tvm.target.target.micro(model="host") + target = tvm.micro.testing.get_target("crt") with tvm.transform.PassContext( opt_level=3, config={"tir.disable_vectorize": True}, disabled_pass=["AlterOpLayout"] ): diff --git a/tests/micro/zephyr/test_zephyr.py b/tests/micro/zephyr/test_zephyr.py index f86f4a7a7f3f..89bd9c75fbe9 100644 --- a/tests/micro/zephyr/test_zephyr.py +++ b/tests/micro/zephyr/test_zephyr.py @@ -40,8 +40,7 @@ def _make_sess_from_op( temp_dir, - model, - zephyr_board, + board, op_name, sched, arg_bufs, @@ -50,23 +49,23 @@ def _make_sess_from_op( serial_number, ): runtime = Runtime("crt", {"system-lib": True}) - target = tvm.target.target.micro(model) + target = tvm.micro.testing.get_target("zephyr", board) target = tvm.target.Target(target=target, host=target) with tvm.transform.PassContext(opt_level=3, config={"tir.disable_vectorize": True}): mod = tvm.build(sched, arg_bufs, target=target, runtime=runtime, name=op_name) - return _make_session(temp_dir, zephyr_board, mod, build_config, use_fvp, serial_number) + return _make_session(temp_dir, board, mod, build_config, use_fvp, serial_number) -def _make_session(temp_dir, zephyr_board, mod, build_config, use_fvp, serial_number): +def _make_session(temp_dir, board, mod, build_config, use_fvp, serial_number): config_main_stack_size = None - if utils.qemu_boards(zephyr_board): + if utils.ZEPHYR_BOARDS[board]["is_qemu"]: config_main_stack_size = 1536 project_options = { "project_type": "host_driven", "verbose": bool(build_config.get("debug")), - "board": zephyr_board, + "board": board, "arm_fvp_path": "/opt/arm/FVP_Corstone_SSE-300/models/Linux64_GCC-6.4/FVP_Corstone_SSE-300_Ethos-U55", "use_fvp": bool(use_fvp), "serial_number": serial_number, @@ -85,17 +84,14 @@ def _make_session(temp_dir, zephyr_board, mod, build_config, use_fvp, serial_num return tvm.micro.Session(project.transport()) -def _make_add_sess( - temp_dir, model, zephyr_board, build_config, use_fvp, serial_number, dtype="int8" -): +def _make_add_sess(temp_dir, board, build_config, use_fvp, serial_number, dtype="int8"): A = tvm.te.placeholder((2,), dtype=dtype) B = tvm.te.placeholder((1,), dtype=dtype) C = tvm.te.compute(A.shape, lambda i: A[i] + B[0], name="C") sched = tvm.te.create_schedule(C.op) return _make_sess_from_op( temp_dir, - model, - zephyr_board, + board, "add", sched, [A, B, C], @@ -111,8 +107,6 @@ def _make_add_sess( @pytest.mark.xfail_on_fvp() def test_add_uint(workspace_dir, board, microtvm_debug, use_fvp, serial_number): """Test compiling the on-device runtime.""" - - model = utils.ZEPHYR_BOARDS[board] build_config = {"debug": microtvm_debug} # NOTE: run test in a nested function so cPython will delete arrays before closing the session. @@ -128,7 +122,7 @@ def test_basic_add(sess): system_lib.get_function("add")(A_data, B_data, C_data) assert (C_data.numpy() == np.array([6, 7])).all() - with _make_add_sess(workspace_dir, model, board, build_config, use_fvp, serial_number) as sess: + with _make_add_sess(workspace_dir, board, build_config, use_fvp, serial_number) as sess: test_basic_add(sess) @@ -138,8 +132,7 @@ def test_basic_add(sess): @pytest.mark.xfail_on_fvp() def test_add_float(workspace_dir, board, microtvm_debug, use_fvp, serial_number): """Test compiling the on-device runtime.""" - model = utils.ZEPHYR_BOARDS[board] - if not utils.has_fpu(board): + if not utils.ZEPHYR_BOARDS[board]["fpu"]: pytest.skip(f"FPU not enabled for {board}") build_config = {"debug": microtvm_debug} @@ -159,7 +152,6 @@ def test_basic_add(sess): with _make_add_sess( workspace_dir, - model, board, build_config, use_fvp, @@ -174,8 +166,6 @@ def test_basic_add(sess): @pytest.mark.xfail_on_fvp() def test_platform_timer(workspace_dir, board, microtvm_debug, use_fvp, serial_number): """Test compiling the on-device runtime.""" - - model = utils.ZEPHYR_BOARDS[board] build_config = {"debug": microtvm_debug} # NOTE: run test in a nested function so cPython will delete arrays before closing the session. @@ -196,7 +186,7 @@ def test_basic_add(sess): assert result.mean > 0 assert len(result.results) == 3 - with _make_add_sess(workspace_dir, model, board, build_config, use_fvp, serial_number) as sess: + with _make_add_sess(workspace_dir, board, build_config, use_fvp, serial_number) as sess: test_basic_add(sess) @@ -205,7 +195,6 @@ def test_basic_add(sess): @pytest.mark.xfail_on_fvp() def test_relay(workspace_dir, board, microtvm_debug, use_fvp, serial_number): """Testing a simple relay graph""" - model = utils.ZEPHYR_BOARDS[board] build_config = {"debug": microtvm_debug} shape = (10,) dtype = "int8" @@ -218,7 +207,7 @@ def test_relay(workspace_dir, board, microtvm_debug, use_fvp, serial_number): ir_mod = tvm.IRModule.from_expr(func) runtime = Runtime("crt", {"system-lib": True}) - target = tvm.target.target.micro(model) + target = tvm.micro.testing.get_target("zephyr", board) with tvm.transform.PassContext(opt_level=3, config={"tir.disable_vectorize": True}): mod = tvm.relay.build(ir_mod, target=target, runtime=runtime) @@ -239,7 +228,6 @@ def test_relay(workspace_dir, board, microtvm_debug, use_fvp, serial_number): @pytest.mark.xfail_on_fvp() def test_onnx(workspace_dir, board, microtvm_debug, use_fvp, serial_number): """Testing a simple ONNX model.""" - model = utils.ZEPHYR_BOARDS[board] build_config = {"debug": microtvm_debug} this_dir = pathlib.Path(os.path.dirname(__file__)) @@ -262,7 +250,7 @@ def test_onnx(workspace_dir, board, microtvm_debug, use_fvp, serial_number): # There is currently a bug preventing the host_driven environment from receiving # the model weights when set using graph_mod.set_input(). # See: https://github.com/apache/tvm/issues/7567 - target = tvm.target.target.micro(model) + target = tvm.micro.testing.get_target("zephyr", board) with tvm.transform.PassContext(opt_level=3, config={"tir.disable_vectorize": True}): executor = Executor("graph", {"link-params": True}) runtime = Runtime("crt", {"system-lib": True}) @@ -292,8 +280,7 @@ def test_onnx(workspace_dir, board, microtvm_debug, use_fvp, serial_number): def check_result( temp_dir, relay_mod, - model, - zephyr_board, + board, map_inputs, out_shape, result, @@ -304,13 +291,11 @@ def check_result( """Helper function to verify results""" TOL = 1e-5 runtime = Runtime("crt", {"system-lib": True}) - target = tvm.target.target.micro(model) + target = tvm.micro.testing.get_target("zephyr", board) with tvm.transform.PassContext(opt_level=3, config={"tir.disable_vectorize": True}): mod = tvm.relay.build(relay_mod, target=target, runtime=runtime) - with _make_session( - temp_dir, zephyr_board, mod, build_config, use_fvp, serial_number - ) as session: + with _make_session(temp_dir, board, mod, build_config, use_fvp, serial_number) as session: rt_mod = tvm.micro.create_local_graph_executor( mod.get_graph_json(), session.get_system_lib(), session.device ) @@ -334,7 +319,6 @@ def check_result( @pytest.mark.xfail_on_fvp() def test_byoc_microtvm(workspace_dir, board, microtvm_debug, use_fvp, serial_number): """This is a simple test case to check BYOC capabilities of microTVM""" - model = utils.ZEPHYR_BOARDS[board] build_config = {"debug": microtvm_debug} x = relay.var("x", shape=(10, 10)) w0 = relay.var("w0", shape=(10, 10)) @@ -387,22 +371,19 @@ def test_byoc_microtvm(workspace_dir, board, microtvm_debug, use_fvp, serial_num ), axis=0, ), - model=model, - zephyr_board=board, + board=board, build_config=build_config, use_fvp=use_fvp, serial_number=serial_number, ) -def _make_add_sess_with_shape( - temp_dir, model, zephyr_board, shape, build_config, use_fvp, serial_number -): +def _make_add_sess_with_shape(temp_dir, board, shape, build_config, use_fvp, serial_number): A = tvm.te.placeholder(shape, dtype="int8") C = tvm.te.compute(A.shape, lambda i: A[i] + A[i], name="C") sched = tvm.te.create_schedule(C.op) return _make_sess_from_op( - temp_dir, model, zephyr_board, "add", sched, [A, C], build_config, use_fvp, serial_number + temp_dir, board, "add", sched, [A, C], build_config, use_fvp, serial_number ) @@ -419,7 +400,6 @@ def _make_add_sess_with_shape( @pytest.mark.xfail_on_fvp() def test_rpc_large_array(workspace_dir, board, microtvm_debug, shape, use_fvp, serial_number): """Test large RPC array transfer.""" - model = utils.ZEPHYR_BOARDS[board] build_config = {"debug": microtvm_debug} # NOTE: run test in a nested function so cPython will delete arrays before closing the session. @@ -432,7 +412,7 @@ def test_tensors(sess): assert (C_data.numpy() == np.zeros(shape)).all() with _make_add_sess_with_shape( - workspace_dir, model, board, shape, build_config, use_fvp, serial_number + workspace_dir, board, shape, build_config, use_fvp, serial_number ) as sess: test_tensors(sess) @@ -445,7 +425,6 @@ def test_autotune_conv2d(workspace_dir, board, microtvm_debug, use_fvp, serial_n pytest.xfail(f"Autotune fails on {board}.") runtime = Runtime("crt", {"system-lib": True}) - model = utils.ZEPHYR_BOARDS[board] build_config = {"debug": microtvm_debug} # Create a Relay model @@ -473,14 +452,14 @@ def test_autotune_conv2d(workspace_dir, board, microtvm_debug, use_fvp, serial_n ).astype("float32") params = {mod["main"].params[1].name_hint: weight_sample} - target = tvm.target.target.micro(model) + target = tvm.micro.testing.get_target("zephyr", board) pass_context = tvm.transform.PassContext(opt_level=3, config={"tir.disable_vectorize": True}) with pass_context: tasks = tvm.autotvm.task.extract_from_program(mod["main"], {}, target) assert len(tasks) > 0 config_main_stack_size = None - if utils.qemu_boards(board): + if utils.ZEPHYR_BOARDS[board]["is_qemu"]: config_main_stack_size = 1536 project_options = { @@ -572,9 +551,10 @@ def test_schedule_build_with_cmsis_dependency(workspace_dir, board, microtvm_deb """Test Relay schedule with CMSIS dependency. This test shows if microTVM Auto tuning with Zephyr breaks if CMSIS dependency was required for a schedule. """ - model = utils.ZEPHYR_BOARDS[board] build_config = {"debug": microtvm_debug} - target = tvm.target.target.micro(model, options=["-keys=arm_cpu,cpu"]) + target = tvm.target.target.micro( + utils.ZEPHYR_BOARDS[board]["model"], options=["-keys=arm_cpu,cpu"] + ) if not target.features.has_dsp: pytest.skip(f"ISA does not support DSP. target: {target}") diff --git a/tests/micro/zephyr/test_zephyr_aot_exec.py b/tests/micro/zephyr/test_zephyr_aot_exec.py index a67cf0830a70..d42c7a00b40e 100644 --- a/tests/micro/zephyr/test_zephyr_aot_exec.py +++ b/tests/micro/zephyr/test_zephyr_aot_exec.py @@ -19,6 +19,7 @@ import tvm import tvm.testing +import tvm.micro.testing import tvm.relay as relay from tvm.relay.backend import Executor, Runtime from tvm.contrib import utils @@ -26,9 +27,9 @@ from . import utils -def _make_session(workspace_dir, zephyr_board, mod, build_config, use_fvp, serial_number): +def _make_session(workspace_dir, board, mod, build_config, use_fvp, serial_number): config_main_stack_size = None - if utils.qemu_boards(zephyr_board): + if utils.ZEPHYR_BOARDS[board]["is_qemu"]: # fyi: qemu_riscv64 seems to be the greediest stack user config_main_stack_size = 4096 else: @@ -38,7 +39,7 @@ def _make_session(workspace_dir, zephyr_board, mod, build_config, use_fvp, seria project_options = { "project_type": "host_driven", "verbose": bool(build_config.get("debug")), - "board": zephyr_board, + "board": board, "arm_fvp_path": "/opt/arm/FVP_Corstone_SSE-300/models/Linux64_GCC-6.4/FVP_Corstone_SSE-300_Ethos-U55", "use_fvp": bool(use_fvp), "serial_number": serial_number, @@ -63,7 +64,6 @@ def _make_session(workspace_dir, zephyr_board, mod, build_config, use_fvp, seria def test_relay(workspace_dir, board, microtvm_debug, use_fvp, serial_number): """Testing a simple relay graph""" - model = utils.ZEPHYR_BOARDS[board] build_config = {"debug": microtvm_debug} shape = (10,) dtype = "int8" @@ -77,7 +77,7 @@ def test_relay(workspace_dir, board, microtvm_debug, use_fvp, serial_number): runtime = Runtime("crt", {"system-lib": True}) executor = Executor("aot") - target = tvm.target.target.micro(model) + target = tvm.micro.testing.get_target("zephyr", board) with tvm.transform.PassContext(opt_level=3, config={"tir.disable_vectorize": True}): mod = tvm.relay.build(ir_mod, target=target, runtime=runtime, executor=executor) @@ -98,7 +98,6 @@ def test_relay(workspace_dir, board, microtvm_debug, use_fvp, serial_number): def test_aot_executor(workspace_dir, board, microtvm_debug, use_fvp, serial_number): """Test use of the AOT executor with microTVM.""" - model = utils.ZEPHYR_BOARDS[board] build_config = {"debug": microtvm_debug} shape = (10,) dtype = "int8" @@ -117,7 +116,7 @@ def @main(%a : Tensor[(1, 2), uint8], %b : Tensor[(1, 2), uint8]) { runtime = Runtime("crt", {"system-lib": True}) executor = Executor("aot") - target = tvm.target.target.micro(model) + target = tvm.micro.testing.get_target("zephyr", board) with tvm.transform.PassContext(opt_level=3, config={"tir.disable_vectorize": True}): mod = tvm.relay.build(relay_mod, target=target, runtime=runtime, executor=executor) diff --git a/tests/micro/zephyr/test_zephyr_aot_exec_standalone.py b/tests/micro/zephyr/test_zephyr_aot_exec_standalone.py index 9e015448e91b..16c1f9e30814 100644 --- a/tests/micro/zephyr/test_zephyr_aot_exec_standalone.py +++ b/tests/micro/zephyr/test_zephyr_aot_exec_standalone.py @@ -22,6 +22,7 @@ import tvm import tvm.testing +import tvm.micro.testing from tvm.micro.project_api import server import tvm.relay as relay from tvm.relay.backend import Executor, Runtime @@ -34,7 +35,6 @@ @pytest.mark.skip_boards(["mps2_an521", "mps3_an547"]) def test_tflite(workspace_dir, board, microtvm_debug, serial_number): """Testing a TFLite model.""" - model = utils.ZEPHYR_BOARDS[board] input_shape = (1, 49, 10, 1) output_shape = (1, 12) build_config = {"debug": microtvm_debug} @@ -58,7 +58,7 @@ def test_tflite(workspace_dir, board, microtvm_debug, serial_number): tflite_model, shape_dict={"input_1": input_shape}, dtype_dict={"input_1 ": "int8"} ) - target = tvm.target.target.micro(model) + target = tvm.micro.testing.get_target("zephyr", board) executor = Executor( "aot", {"unpacked-api": True, "interface-api": "c", "workspace-byte-alignment": 4} ) @@ -90,10 +90,9 @@ def test_tflite(workspace_dir, board, microtvm_debug, serial_number): @pytest.mark.skip_boards(["mps2_an521", "mps3_an547"]) def test_qemu_make_fail(workspace_dir, board, microtvm_debug, serial_number): """Testing QEMU make fail.""" - if board not in ["qemu_x86", "mps2_an521", "mps3_an547"]: + if not utils.ZEPHYR_BOARDS[board]["is_qemu"]: pytest.skip(msg="Only for QEMU targets.") - model = utils.ZEPHYR_BOARDS[board] build_config = {"debug": microtvm_debug} shape = (10,) dtype = "float32" @@ -105,7 +104,7 @@ def test_qemu_make_fail(workspace_dir, board, microtvm_debug, serial_number): func = relay.Function([x], z) ir_mod = tvm.IRModule.from_expr(func) - target = tvm.target.target.micro(model) + target = tvm.micro.testing.get_target("zephyr", board) executor = Executor("aot") runtime = Runtime("crt") with tvm.transform.PassContext(opt_level=3, config={"tir.disable_vectorize": True}): diff --git a/tests/micro/zephyr/test_zephyr_armv7m.py b/tests/micro/zephyr/test_zephyr_armv7m.py index eb709382024d..cd589a19e886 100644 --- a/tests/micro/zephyr/test_zephyr_armv7m.py +++ b/tests/micro/zephyr/test_zephyr_armv7m.py @@ -95,21 +95,12 @@ def _apply_desired_layout_no_simd(relay_mod): @tvm.testing.requires_micro -@pytest.mark.skip_boards(["mps2_an521"]) +@pytest.mark.skip_boards( + ["mps2_an521", "stm32f746g_disco", "nucleo_f746zg", "nucleo_l4r5zi", "nrf5340dk_nrf5340_cpuapp"] +) @pytest.mark.xfail(reason="due https://github.com/apache/tvm/issues/12619") def test_armv7m_intrinsic(workspace_dir, board, microtvm_debug, serial_number): """Testing a ARM v7m SIMD extension.""" - if board not in [ - "mps2_an521", - "stm32f746g_disco", - "nucleo_f746zg", - "nucleo_l4r5zi", - "nrf5340dk_nrf5340_cpuapp", - ]: - pytest.skip(msg="Platform does not support ARM v7m SIMD extension.") - - model = utils.ZEPHYR_BOARDS[board] - build_config = {"debug": microtvm_debug} this_dir = pathlib.Path(os.path.dirname(__file__)) @@ -123,8 +114,10 @@ def test_armv7m_intrinsic(workspace_dir, board, microtvm_debug, serial_number): # kernel layout "HWIO" is not supported by arm_cpu SIMD extension (see tvm\python\relay\op\strategy\arm_cpu.py) relay_mod_no_simd = _apply_desired_layout_no_simd(relay_mod) - target = tvm.target.target.micro(model, options=["-keys=cpu"]) - target_simd = tvm.target.target.micro(model, options=["-keys=arm_cpu,cpu"]) + target = tvm.target.target.micro(utils.ZEPHYR_BOARDS[board]["model"], options=["-keys=cpu"]) + target_simd = tvm.target.target.micro( + utils.ZEPHYR_BOARDS[board]["model"], options=["-keys=arm_cpu,cpu"] + ) executor = Executor("aot", {"unpacked-api": True, "interface-api": "c"}) runtime = Runtime("crt") diff --git a/tests/micro/zephyr/utils.py b/tests/micro/zephyr/utils.py index bdac4e9c63a7..26f9d6a10e2d 100644 --- a/tests/micro/zephyr/utils.py +++ b/tests/micro/zephyr/utils.py @@ -41,41 +41,19 @@ TEMPLATE_PROJECT_DIR = pathlib.Path(tvm.micro.get_microtvm_template_projects("zephyr")) -BOARDS = TEMPLATE_PROJECT_DIR / "boards.json" - _LOG = logging.getLogger(__name__) def zephyr_boards() -> dict: - """Returns a dict mapping board to target model""" - with open(BOARDS) as f: + """Returns Zephyr board properties""" + with open(TEMPLATE_PROJECT_DIR / "boards.json") as f: board_properties = json.load(f) - - boards_model = {board: info["model"] for board, info in board_properties.items()} - return boards_model + return board_properties ZEPHYR_BOARDS = zephyr_boards() -def qemu_boards(board: str): - """Returns True if board is QEMU.""" - with open(BOARDS) as f: - board_properties = json.load(f) - - qemu_boards = [name for name, board in board_properties.items() if board["is_qemu"]] - return board in qemu_boards - - -def has_fpu(board: str): - """Returns True if board has FPU.""" - with open(BOARDS) as f: - board_properties = json.load(f) - - fpu_boards = [name for name, board in board_properties.items() if board["fpu"]] - return board in fpu_boards - - def build_project( temp_dir, zephyr_board, mod, build_config, serial_number, simd=False, extra_files_tar=None ):