diff --git a/docs/script_convert.py b/docs/script_convert.py new file mode 100644 index 000000000000..edd173b2956e --- /dev/null +++ b/docs/script_convert.py @@ -0,0 +1,94 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import argparse +import pathlib + +BASH = "# bash" +BASH_IGNORE = "# bash-ignore" +BASH_MULTILINE_COMMENT_START = ": '" +BASH_MULTILINE_COMMENT_END = "'" + + +def bash_to_python(src_path: pathlib.Path, dest_path: pathlib.Path): + """Convert a bash script file to a Python format compatible with Sphinx doc.""" + with open(src_path, "r") as src_f: + with open(dest_path, "w") as dest_f: + line = src_f.readline() + bash_block = [] + bash_detected = False + bash_ignore_detected = False + new_line_required = False + while line: + line = line.strip("\n").strip("\r") + if bash_detected: + if line == BASH: + # write the bash block to destination + if new_line_required: + dest_f.write("\n") + python_code = "# .. code-block:: bash\n#\n" + for bash_line in bash_block: + python_code += f"#\t {bash_line}\n" + python_code += "#" + dest_f.write(python_code) + + bash_detected = False + bash_block = [] + new_line_required = True + else: + # add new bash command line + bash_block.append(line) + elif bash_ignore_detected: + if line == BASH_IGNORE: + bash_ignore_detected = False + new_line_required = True + else: + new_line_required = False + pass + else: + if line == BASH: + bash_detected = True + elif line == BASH_IGNORE: + bash_ignore_detected = True + elif line in [BASH_MULTILINE_COMMENT_START, BASH_MULTILINE_COMMENT_END]: + if new_line_required: + dest_f.write("\n") + dest_f.write('"""') + new_line_required = True + else: + if new_line_required: + dest_f.write("\n") + dest_f.write(f"{line}") + new_line_required = True + + line = src_f.readline() + if new_line_required: + dest_f.write("\n") + + +def main(): + parser = argparse.ArgumentParser(description="Convert tutorial script to Python.") + parser.add_argument("script", type=str, help="Path to script file.") + args = parser.parse_args() + + src_path = pathlib.Path(args.script) + dest_path = src_path.parent / f"{src_path.stem}.py" + bash_to_python(src_path, dest_path) + + +if __name__ == "__main__": + main() diff --git a/gallery/how_to/work_with_microtvm/micro_tvmc.py b/gallery/how_to/work_with_microtvm/micro_tvmc.sh old mode 100644 new mode 100755 similarity index 85% rename from gallery/how_to/work_with_microtvm/micro_tvmc.py rename to gallery/how_to/work_with_microtvm/micro_tvmc.sh index 423e0f1dde37..0b789216f21b --- a/gallery/how_to/work_with_microtvm/micro_tvmc.py +++ b/gallery/how_to/work_with_microtvm/micro_tvmc.sh @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -""" +: ' .. _tutorial-micro-tvmc: Executing a Tiny Model with TVMC Micro @@ -25,7 +25,7 @@ This tutorial explains how to compile a tiny model for a micro device, build a program on Zephyr platform to execute this model, flash the program and run the model all using `tvmc micro` command. -""" +' ###################################################################### # .. note:: @@ -42,6 +42,10 @@ # ./docker/bash.sh tlcpack/ci-qemu # +# bash-ignore +shopt -s expand_aliases +alias tvmc="python3 -m tvm.driver.tvmc" +# bash-ignore ############################################################ # Using TVMC Micro @@ -55,16 +59,15 @@ # # To check if you have TVMC command installed on your machine, you can run: # -# .. code-block:: bash -# -# tvmc --help -# +# bash +tvmc --help +# bash # To compile a model for microtvm we use ``tvmc compile`` subcommand. The output of this command # is used in next steps with ``tvmc micro`` subcommands. You can check the availability of TVMC Micro using: # -# .. code-block:: bash -# -# tvmc micro --help +# bash +tvmc micro --help +# bash # # The main tasks that you can perform using ``tvmc micro`` are ``create``, ``build`` and ``flash``. # To read about specific options under a givern subcommand, use @@ -80,10 +83,9 @@ # # For this tutorial we will be using the model in tflite format. # -# .. code-block:: bash -# -# wget https://github.com/tensorflow/tflite-micro/raw/main/tensorflow/lite/micro/examples/magic_wand/magic_wand.tflite -# +# bash +wget https://github.com/tensorflow/tflite-micro/raw/main/tensorflow/lite/micro/examples/magic_wand/magic_wand.tflite +# bash ############################################################ # Compiling a TFLite model to a Model Library Format @@ -95,19 +97,18 @@ # # Here, we generate a MLF file for ``qemu_x86`` Zephyr board. To generate MLF output for the ``magic_wand`` tflite model: # -# .. code-block:: bash -# -# tvmc compile magic_wand.tflite \ -# --target='c -keys=cpu -link-params=0 -model=host' \ -# --runtime=crt \ -# --runtime-crt-system-lib 1 \ -# --executor='graph' \ -# --executor-graph-link-params 0 \ -# --output model.tar \ -# --output-format mlf \ -# --pass-config tir.disable_vectorize=1 \ -# --disabled-pass=AlterOpLayout -# +# bash +tvmc compile magic_wand.tflite \ + --target='c -keys=cpu -link-params=0 -model=host' \ + --runtime=crt \ + --runtime-crt-system-lib 1 \ + --executor='graph' \ + --executor-graph-link-params 0 \ + --output model.tar \ + --output-format mlf \ + --pass-config tir.disable_vectorize=1 \ + --disabled-pass=AlterOpLayout +# bash # This will generate a ``model.tar`` file which contains TVM compiler output files. To run this command for # a different Zephyr device, you need to update ``target``. For instance, for ``nrf5340dk_nrf5340_cpuapp`` board # the target is ``--target='c -keys=cpu -link-params=0 -model=nrf5340dk'``. @@ -122,14 +123,13 @@ # for the project to ``create`` subcommand along with project options. Project options for each # platform (Zephyr/Arduino) are defined in their Project API server file. To generate Zephyr project, run: # -# .. code-block:: bash -# -# tvmc micro create \ -# project \ -# model.tar \ -# zephyr \ -# --project-option project_type=host_driven zephyr_board=qemu_x86 -# +# bash +tvmc micro create \ + project \ + model.tar \ + zephyr \ + --project-option project_type=host_driven zephyr_board=qemu_x86 +# bash # This will generate a ``Host-Driven`` Zephyr project for ``qemu_x86`` Zephyr board. In Host-Driven template project, # the Graph Executor will run on host and perform the model execution on Zephyr device by issuing commands to the # device using an RPC mechanism. Read more about `Host-Driven Execution `_. @@ -148,26 +148,24 @@ # Next step is to build the Zephyr project which includes TVM generated code for running the tiny model, Zephyr # template code to run a model in Host-Driven mode and TVM runtime source/header files. To build the project: # -# .. code-block:: bash -# -# tvmc micro build \ -# project \ -# zephyr \ -# --project-option zephyr_board=qemu_x86 -# +# bash +tvmc micro build \ + project \ + zephyr \ + --project-option zephyr_board=qemu_x86 +# bash # This will build the project in ``project`` directory and generates binary files under ``project/build``. To build # Zephyr project for a different Zephyr board, change ``zephyr_board`` project option. # # Next, we flash the Zephyr binary file to Zephyr device. For ``qemu_x86`` Zephyr board this step does not # actually perform any action since QEMU will be used, however you need this step for physical hardware. # -# .. code-block:: bash -# -# tvmc micro flash \ -# project \ -# zephyr \ -# --project-option zephyr_board=qemu_x86 -# +# bash +tvmc micro flash \ + project \ + zephyr \ + --project-option zephyr_board=qemu_x86 +# bash ############################################################ # Run Tiny Model on Micro Target @@ -179,14 +177,14 @@ # and pass ``--device micro`` to specify the device type. This command will open a communication channel, set input # values using ``Graph Executor`` on host and run full model on the device. Then it gets output from the device. # -# .. code-block:: bash -# -# tvmc run \ -# --device micro \ -# project \ -# --project-option zephyr_board=qemu_x86 \ -# --fill-mode ones -# --print-top 4 +# bash +tvmc run \ + --device micro \ + project \ + --project-option zephyr_board=qemu_x86 \ + --fill-mode ones \ + --print-top 4 +# bash # # Output: # # # # INFO:__main__:b'[100%] [QEMU] CPU: qemu32,+nx,+pae\n' diff --git a/tests/python/unittest/test_ci.py b/tests/python/ci/test_ci.py similarity index 99% rename from tests/python/unittest/test_ci.py rename to tests/python/ci/test_ci.py index 645f239f9abc..4c7ad7cf341e 100644 --- a/tests/python/unittest/test_ci.py +++ b/tests/python/ci/test_ci.py @@ -15,16 +15,13 @@ # specific language governing permissions and limitations # under the License. -import pathlib import subprocess import sys import json import textwrap -import tempfile - import pytest -REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent.parent.parent +from test_utils import REPO_ROOT class TempGit: diff --git a/tests/python/ci/test_script_converter.py b/tests/python/ci/test_script_converter.py new file mode 100644 index 000000000000..a792c135811e --- /dev/null +++ b/tests/python/ci/test_script_converter.py @@ -0,0 +1,162 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import sys + +import pytest + +from tvm.contrib import utils + +from test_utils import REPO_ROOT + +sys.path.insert(0, str(REPO_ROOT / "docs")) +from script_convert import ( + bash_to_python, + BASH, + BASH_IGNORE, + BASH_MULTILINE_COMMENT_START, + BASH_MULTILINE_COMMENT_END, +) + + +def test_bash_cmd(): + temp = utils.tempdir() + src_path = temp / "src.sh" + dest_path = temp / "dest.py" + + with open(src_path, "w") as src_f: + src_f.write(BASH) + src_f.write("\n") + src_f.write("tvmc\n") + src_f.write(BASH) + src_f.write("\n") + + bash_to_python(src_path, dest_path) + + with open(dest_path, "r") as dest_f: + generated_cmd = dest_f.read() + + expected_cmd = "# .. code-block:: bash\n" "#\n" "#\t tvmc\n" "#\n" + + assert generated_cmd == expected_cmd + + +def test_bash_ignore_cmd(): + temp = utils.tempdir() + src_path = temp / "src.sh" + dest_path = temp / "dest.py" + + with open(src_path, "w") as src_f: + src_f.write("# start\n") + src_f.write(BASH_IGNORE) + src_f.write("\n") + src_f.write("tvmc\n") + src_f.write(BASH_IGNORE) + src_f.write("\n") + src_f.write("# end\n") + + bash_to_python(src_path, dest_path) + + with open(dest_path, "r") as dest_f: + generated_cmd = dest_f.read() + + expected_cmd = "# start\n" "# end\n" + assert generated_cmd == expected_cmd + + +def test_no_command(): + temp = utils.tempdir() + src_path = temp / "src.sh" + dest_path = temp / "dest.py" + + with open(src_path, "w") as src_f: + src_f.write("# start\n") + src_f.write("# description\n") + src_f.write("end\n") + + bash_to_python(src_path, dest_path) + + with open(dest_path, "r") as dest_f: + generated_cmd = dest_f.read() + + expected_cmd = "# start\n" "# description\n" "end\n" + assert generated_cmd == expected_cmd + + +def test_text_and_bash_command(): + temp = utils.tempdir() + src_path = temp / "src.sh" + dest_path = temp / "dest.py" + + with open(src_path, "w") as src_f: + src_f.write("# start\n") + src_f.write(BASH) + src_f.write("\n") + src_f.write("tvmc\n") + src_f.write(BASH) + src_f.write("\n") + src_f.write("# end\n") + + bash_to_python(src_path, dest_path) + + with open(dest_path, "r") as dest_f: + generated_cmd = dest_f.read() + + expected_cmd = "# start\n" "# .. code-block:: bash\n" "#\n" "#\t tvmc\n" "#\n" "# end\n" + + assert generated_cmd == expected_cmd + + +def test_last_line_break(): + temp = utils.tempdir() + src_path = temp / "src.sh" + dest_path = temp / "dest.py" + + with open(src_path, "w") as src_f: + src_f.write("# start\n") + src_f.write("# end\n") + + bash_to_python(src_path, dest_path) + + with open(dest_path, "r") as dest_f: + generated_cmd = dest_f.read() + + expected_cmd = "# start\n" "# end\n" + + assert generated_cmd == expected_cmd + + +def test_multiline_comment(): + temp = utils.tempdir() + src_path = temp / "src.sh" + dest_path = temp / "dest.py" + + with open(src_path, "w") as src_f: + src_f.write(BASH_MULTILINE_COMMENT_START) + src_f.write("\n") + src_f.write("comment\n") + src_f.write(BASH_MULTILINE_COMMENT_END) + src_f.write("\n") + + bash_to_python(src_path, dest_path) + + with open(dest_path, "r") as dest_f: + generated_cmd = dest_f.read() + + expected_cmd = '"""\n' "comment\n" '"""\n' + + assert generated_cmd == expected_cmd diff --git a/tests/python/ci/test_utils.py b/tests/python/ci/test_utils.py new file mode 100644 index 000000000000..0ad88f19f4cd --- /dev/null +++ b/tests/python/ci/test_utils.py @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import pathlib + +REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent.parent.parent diff --git a/tests/scripts/task_convert_scripts_to_python.sh b/tests/scripts/task_convert_scripts_to_python.sh new file mode 100755 index 000000000000..b2164cbdbbcf --- /dev/null +++ b/tests/scripts/task_convert_scripts_to_python.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +SCRIPTS_DIR=$(dirname "${BASH_SOURCE[0]}") +TVM_DIR=$(cd "${SCRIPTS_DIR}" && git rev-parse --show-toplevel) +python3 "${TVM_DIR}/docs/script_convert.py" "${TVM_DIR}/gallery/how_to/work_with_microtvm/micro_tvmc.sh" diff --git a/tests/scripts/task_lint.sh b/tests/scripts/task_lint.sh index 0a01bea96bb1..a6021e60b65d 100755 --- a/tests/scripts/task_lint.sh +++ b/tests/scripts/task_lint.sh @@ -25,6 +25,9 @@ cleanup() trap cleanup 0 +echo "Convert scripts to Python..." +tests/scripts/task_convert_scripts_to_python.sh + echo "Checking file types..." python3 tests/lint/check_file_type.py diff --git a/tests/scripts/task_python_docs.sh b/tests/scripts/task_python_docs.sh index 926628092074..b947c65ec6cc 100755 --- a/tests/scripts/task_python_docs.sh +++ b/tests/scripts/task_python_docs.sh @@ -58,6 +58,9 @@ sphinx_precheck() { function join_by { local IFS="$1"; shift; echo "$*"; } +# Convert bash tutorials to Python format +tests/scripts/task_convert_scripts_to_python.sh + # These warnings are produced during the docs build for various reasons and are # known to not signficantly affect the output. Don't add anything new to this # list without special consideration of its effects, and don't add anything with diff --git a/tests/scripts/task_python_microtvm.sh b/tests/scripts/task_python_microtvm.sh index 4fb303e42702..d13ee91a0ba8 100755 --- a/tests/scripts/task_python_microtvm.sh +++ b/tests/scripts/task_python_microtvm.sh @@ -41,9 +41,10 @@ run_pytest ctypes python-microtvm-stm32 tests/micro/stm32 run_pytest ctypes python-microtvm-common-qemu_x86 tests/micro/common --board=qemu_x86 run_pytest ctypes python-microtvm-common-due tests/micro/common --test-build-only --board=due -# # Tutorials running with host CRT +# Tutorials python3 gallery/how_to/work_with_microtvm/micro_tflite.py python3 gallery/how_to/work_with_microtvm/micro_autotune.py +./gallery/how_to/work_with_microtvm/micro_tvmc.sh # Tutorials running with Zephyr export TVM_MICRO_USE_HW=1