Merge branch 'pers-upstream' of https://github.com/dchauhan-arm/tvm into pers-upstream

Author: dchauhan-arm

.asf.yaml

@@ -46,3 +46,4 @@ github:
   # cycle. PMC may review and recycle slots when necessary.
   collaborators:
     - denise-k
+    - tvm-bot  # For automated feedback in PR review.

apps/cpp_rpc/rpc_server.cc

@@ -86,24 +86,33 @@ static std::string getNextString(std::stringstream* iss) {
 
 /*!
  * \brief RPCServer RPC Server class.
+ *
  * \param host The hostname of the server, Default=0.0.0.0
- * \param port The port of the RPC, Default=9090
- * \param port_end The end search port of the RPC, Default=9099
- * \param tracker The address of RPC tracker in host:port format e.g. 10.77.1.234:9190 Default=""
- * \param key The key used to identify the device type in tracker. Default=""
- * \param custom_addr Custom IP Address to Report to RPC Tracker. Default=""
+ *
+ * \param port_search_start The low end of the search range for an
+ *     available port for the RPC, Default=9090
+ *
+ * \param port_search_end The high search the search range for an
+ *     available port for the RPC, Default=9099
+ *
+ * \param tracker The address of RPC tracker in host:port format
+ *     (e.g. "10.77.1.234:9190")
+ *
+ * \param key The key used to identify the device type in tracker.
+ *
+ * \param custom_addr Custom IP Address to Report to RPC Tracker.
  */
 class RPCServer {
  public:
   /*!
    * \brief Constructor.
    */
-  RPCServer(std::string host, int port, int port_end, std::string tracker_addr, std::string key,
-            std::string custom_addr, std::string work_dir)
+  RPCServer(std::string host, int port_search_start, int port_search_end, std::string tracker_addr,
+            std::string key, std::string custom_addr, std::string work_dir)
       : host_(std::move(host)),
-        port_(port),
+        port_search_start_(port_search_start),
         my_port_(0),
-        port_end_(port_end),
+        port_search_end_(port_search_end),
         tracker_addr_(std::move(tracker_addr)),
         key_(std::move(key)),
         custom_addr_(std::move(custom_addr)),
@@ -126,7 +135,7 @@ class RPCServer {
    */
   void Start() {
     listen_sock_.Create();
-    my_port_ = listen_sock_.TryBindHost(host_, port_, port_end_);
+    my_port_ = listen_sock_.TryBindHost(host_, port_search_start_, port_search_end_);
     LOG(INFO) << "bind to " << host_ << ":" << my_port_;
     listen_sock_.Listen(1);
     std::future<void> proc(std::async(std::launch::async, &RPCServer::ListenLoopProc, this));
@@ -140,7 +149,7 @@ class RPCServer {
    * \brief ListenLoopProc The listen process.
    */
   void ListenLoopProc() {
-    TrackerClient tracker(tracker_addr_, key_, custom_addr_, port_);
+    TrackerClient tracker(tracker_addr_, key_, custom_addr_, my_port_);
     while (true) {
       support::TCPSocket conn;
       support::SockAddr addr("0.0.0.0", 0);
@@ -340,9 +349,9 @@ class RPCServer {
   }
 
   std::string host_;
-  int port_;
+  int port_search_start_;
   int my_port_;
-  int port_end_;
+  int port_search_end_;
   std::string tracker_addr_;
   std::string key_;
   std::string custom_addr_;

cmake/config.cmake

@@ -231,6 +231,10 @@ set(USE_ETHOSN_HW OFF)
 # Whether to build with Arm(R) Ethos(TM)-U NPU codegen support
 set(USE_ETHOSU OFF)
 
+# Whether to build with CMSIS-NN external library support.
+# See https://github.com/ARM-software/CMSIS_5
+set(USE_CMSISNN OFF)
+
 # Whether to build with TensorRT codegen or runtime
 # Examples are available here: docs/deploy/tensorrt.rst.
 #

docker/install/ubuntu_install_vela.sh

@@ -20,7 +20,4 @@ set -e
 set -u
 set -o pipefail
 
-# In a refactor between v2.1.1 and v3.0.0, find_block_configs <appropriate function name> was removed from Vela.
-# Since this is still required for the TVM port, it will be reinstated in Vela in a future release.
-# Until then, it needs to be pinned to v2.1.1.
-pip3 install ethos-u-vela==2.1.1
+pip3 install ethos-u-vela==3.2.0

docs/arch/index.rst

@@ -423,4 +423,5 @@ microTVM
    :maxdepth: 1
 
    microtvm_design
+   microtvm_project_api
    model_library_format

docs/arch/microtvm_project_api.rst

@@ -0,0 +1,150 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+.. _microtvm_project_api:
+
+microTVM Project API
+====================
+
+About microTVM Project API
+--------------------------
+
+The microTVM Project API allows TVM to automatically run models on
+unconventional or embedded platforms. It allows platforms to define a standard
+function to integrate TVM compiler output with boilerplate platform-specific
+code, producing a runnable **Project**. Project API then further defines
+functions to build that project, program compatible devices accessible from the
+TVM machine, and communicate with the running code so that TVM can perform
+host-driven inference and autotuning.
+
+There are many cases where it might be desirable simply to invoke microTVM as a
+tool from your platform's build process. Indeed, for the average firmware
+developer, this is likely to be all they need. However, there are a couple of
+use cases when you may want to teach microTVM how to build firmware using your
+platform's build tool:
+
+1.  To enable AutoTVM and AutoScheduling on your platform. Defining a Project
+    API implementation allows TVM to tune models for peak performance on your
+    platform.
+2.  To enable engineers without firmware expertise to experiment with models on
+    your platform. Defining a Project API implementation allows these engineers
+    to leverage the standard TVM Python workflows to perform host-driven
+    inference on your platform.
+3.  Integration Testing. Defining a Project API implementation allows you to
+    create Continuous Integration Tests which verify model correctness and
+    performance on your platform.
+
+API Definition
+--------------
+
+The full API is the ``abstractmethod`` defined on ``ProjectAPIHandler`` in
+`python/tvm/micro/project_api/server.py <https://github.com/apache/tvm/blob/main/python/tvm/micro/project_api/server.py>`_.
+Rather than duplicate the documentation here, we simply refer you to that class.
+
+How TVM uses Project API
+------------------------
+
+This section explains how the Project API should be used with TVM. Project API
+is defined around the *Project* as the buildable unit of firmware. TVM expects
+to be provided initially with a directory containing a *Template Project*, which
+together with a :ref:`Model Library Format <model_library_format>` file can be
+built into a runnable project.
+
+Inside the Template Directory is (typically) a Python script implementing the
+API server. TVM launches this script in a subprocess and sends commands to the
+server to perform each of the actions outlined above.
+
+The typical usage flow is as follows:
+
+1. Launch Project API server in Template Project.
+2. Verify the API server is version-compatible with TVM, plus read properties
+   of the implementation, by sending ``server_info_query`` command.
+3. Generate a new project by sending command ``generate_project`` to create a
+   new project. The arguments to this command is a Model Library Format and a
+   non-existent directory which should be populated with the generated
+   project. The Template Project API server should copy itself into the
+   newly-generated project.
+4. Terminate the Template Project API server.
+5. Launch Project API server in Generated Project.
+6. Verify the API server is version-compatible with TVM, plus read properties
+   of the implementation, by sending ``server_info_query`` command.
+7. Build and flash the projec by sending commands ``build`` and ``flash`` to the
+   API server.
+8. Communicate with the target. Send command ``open_transport`` followed by
+   commands ``write_transport`` and ``read_transport`` to write and read from
+   e.g. a serial port attached to the target. Upon completion,
+   ``close_transport`` is sent.
+9. Terminate Project API server.
+
+Disk Layout of the Project
+--------------------------
+
+In the root directory of a project (template or generated), one of the following
+two files must exist:
+
+- ``microtvm_api_server.py`` - the suggested approach. Place a
+  python3-compatible Python script in the root directory. TVM will execute this
+  script in its own process using the same interpreter used to execute TVM.
+- ``microtvm_api_server.sh`` (on Windows, ``microtvm_api_server.bat``) -
+  alternate approach. When a different Python interpreter is necessary, or
+  when you want to implement the server in a different language, create this
+  executable file. TVM will launch this file in a separate process.
+
+Aside from these two files, no other restrictions are made on the layout.
+
+Communication between TVM and Project API Server
+------------------------------------------------
+
+TVM communicates with the Project API server using `JSON-RPC 2.0
+<https://www.jsonrpc.org/specification>`_. TVM always launches API servers using
+the following command-line:
+
+``microtvm_api_server.py --read-fd <n> --write-fd <n>``
+
+Commands are sent from TVM to the server over the file descriptor given by
+``--read-fd`` and replies are received by TVM from the server over the file
+descriptor given by ``--write-fd``.
+
+Helpers for Implementing the API server in Python
+-------------------------------------------------
+
+TVM provides helper utilities that make it easy to implement the server in Python.
+To implement the server in Python, create ``microtvm_api_server.py`` and add
+``from tvm.micro.project_api import server`` (or, copy this file into your template
+project--there are no dependencies--and import it there instead). Next, subclass
+``ProjectAPIHander``::
+
+    class Handler(server.ProjectAPIHandler):
+        def server_info_query(self, tvm_version):
+            # Implement server_info_query
+
+        def generate_project(self, model_library_format_path, standalone_crt_dir, project_dir, options):
+            # Implement generate_project
+
+        # ...
+
+Finally, invoke the helper ``main()``::
+
+    if __name__ == "__main__":
+        server.main(Handler())
+
+Using Project API from ``tvmc``
+-------------------------------
+
+Each major Project API command is available through the ``tvmc micro``
+sub-command to make debugging interactions simple. Invoke ``tvmc micro --help``
+for more information.

docs/arch/model_library_format.rst

@@ -15,6 +15,8 @@
     specific language governing permissions and limitations
     under the License.
 
+.. _model_library_format:
+
 Model Library Format
 ====================
 

docs/legacy_redirect.py

@@ -104,6 +104,7 @@
     ["vta/dev/index.html", "../../topic/vta/dev/index.html"],
     ["vta/index.html", "../topic/vta/index.html"],
     ["vta/install.html", "../topic/vta/install.html"],
+    ["tutorials/index.html", "../tutorial/index.html"],
     ["tutorials/frontend/from_caffe2.html", "../../how_to/compile_models/from_caffe2.html"],
     ["tutorials/frontend/from_coreml.html", "../../how_to/compile_models/from_coreml.html"],
     ["tutorials/frontend/from_darknet.html", "../../how_to/compile_models/from_darknet.html"],

include/tvm/tir/usmp/utils.h

@@ -153,6 +153,45 @@ class BufferInfo : public ObjectRef {
   TVM_DEFINE_MUTABLE_OBJECT_REF_METHODS(BufferInfo, ObjectRef, BufferInfoNode);
 };
 
+/*!
+ * \brief This is a composite node that is produced by extract_buffer_info
+ * analysis pass that contains useful global information that could be useful
+ * for memory planning algorithms.
+ */
+struct BufferInfoAnalysisNode : public Object {
+  /*! \brief The BufferInfo object and its associated TIR statement */
+  Map<BufferInfo, tir::Stmt> buffer_info_stmts;
+  /*! \brief This represent maximum amount of memory being used at
+   * any point of time in the inference. This value is largely the
+   * best allocation an algorithm could achieve. Due to
+   * the complexities of conflict graphs, it would not be feasible
+   * to achieve this value, practically. However, it can be useful
+   * for iterative algorithms to know this value to define termination
+   * criteria.*/
+  Integer memory_pressure;
+
+  void VisitAttrs(tvm::AttrVisitor* v) {
+    v->Visit("buffer_info_stmts", &buffer_info_stmts);
+    v->Visit("memory_pressure", &memory_pressure);
+  }
+
+  bool SEqualReduce(const BufferInfoAnalysisNode* other, SEqualReducer equal) const {
+    return equal(buffer_info_stmts, other->buffer_info_stmts) &&
+           equal(memory_pressure, other->memory_pressure);
+  }
+
+  void SHashReduce(SHashReducer hash_reduce) const {
+    hash_reduce(buffer_info_stmts);
+    hash_reduce(memory_pressure);
+  }
+};
+
+class BufferInfoAnalysis : public ObjectRef {
+ public:
+  TVM_DLL BufferInfoAnalysis(Map<BufferInfo, tir::Stmt> buffer_info_stmts, Integer memory_pressure);
+  TVM_DEFINE_MUTABLE_OBJECT_REF_METHODS(BufferInfoAnalysis, ObjectRef, BufferInfoAnalysisNode);
+};
+
 /*!
  * \brief The pool allocation produced after the USMP algorithm
  */

python/gen_requirements.py

@@ -109,6 +109,7 @@
             [
                 "future",  # Hidden dependency of torch.
                 "onnx",
+                "onnxoptimizer",
                 "onnxruntime",
                 "torch",
                 "torchvision",
@@ -146,6 +147,7 @@
                 "ethos-u-vela",
                 "future",  # Hidden dependency of torch.
                 "onnx",
+                "onnxoptimizer",
                 "onnxruntime",
                 "paddlepaddle",
                 "tensorflow",
@@ -233,13 +235,14 @@
         "docutils",
         "<0.17",
     ),  # Work around https://github.com/readthedocs/sphinx_rtd_theme/issues/1115
-    ("ethos-u-vela", "==2.1.1"),
+    ("ethos-u-vela", "==3.2.0"),
     ("future", None),
     ("h5py", "==2.10.0"),
     ("image", None),
     ("matplotlib", None),
     ("numpy", None),
     ("onnx", None),
+    ("onnxoptimizer", None),
     ("onnxruntime", None),
     ("opencv-python", None),
     ("paddlepaddle", None),