[microTVM][CRT]Separate CRT template project from standalone CRT build (#13812)

mehrdadh · web-flow · commit b77d24c384b1 · 2023-01-23T10:12:27.000-08:00
This PR separates the CRT template project for microTVM from CRT standalone library build. The CRT template project would move to build/microtvm_template_projects.

It also adds uploading microtvm_template_projects to S3 in few images that were missing this previously.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -481,6 +481,7 @@ include(cmake/utils/CCache.cmake)
 # Module rules
 include(cmake/modules/VTA.cmake)
 include(cmake/modules/StandaloneCrt.cmake)
+include(cmake/modules/CRT.cmake)
 include(cmake/modules/Zephyr.cmake)
 include(cmake/modules/Arduino.cmake)
 include(cmake/modules/CUDA.cmake)
@@ -576,6 +577,7 @@ include(cmake/modules/contrib/PAPI.cmake)
 if(USE_MICRO)
   # NOTE: cmake doesn't track dependencies at the file level across subdirectories. For the
   # Unix Makefiles generator, need to add these explicit target-level dependency)
+  add_dependencies(tvm_runtime crt)
   add_dependencies(tvm_runtime zephyr)
   add_dependencies(tvm_runtime arduino)
   if(MSVC)
diff --git a/ci/jenkins/generated/arm_jenkinsfile.groovy b/ci/jenkins/generated/arm_jenkinsfile.groovy
diff --git a/ci/jenkins/generated/cpu_jenkinsfile.groovy b/ci/jenkins/generated/cpu_jenkinsfile.groovy
diff --git a/ci/jenkins/generated/i386_jenkinsfile.groovy b/ci/jenkins/generated/i386_jenkinsfile.groovy
diff --git a/ci/jenkins/generated/minimal_jenkinsfile.groovy b/ci/jenkins/generated/minimal_jenkinsfile.groovy
diff --git a/ci/jenkins/templates/arm_jenkinsfile.groovy.j2 b/ci/jenkins/templates/arm_jenkinsfile.groovy.j2
@@ -31,7 +31,7 @@
   cmake_build(ci_arm, 'build', '-j4')
   make_standalone_crt(ci_arm, 'build')
   make_cpp_tests(ci_arm, 'build')
-  {{ m.upload_artifacts(tag='arm', filenames=tvm_multilib + cpptest + crttest + standalone_crt) }}
+  {{ m.upload_artifacts(tag='arm', filenames=tvm_multilib + cpptest + crttest + standalone_crt + microtvm_template_projects) }}
 {% endcall %}
 
 {% set test_method_names = [] %}
diff --git a/ci/jenkins/templates/cpu_jenkinsfile.groovy.j2 b/ci/jenkins/templates/cpu_jenkinsfile.groovy.j2
@@ -31,7 +31,7 @@
   cmake_build(ci_cpu, 'build', '-j2')
   make_standalone_crt(ci_cpu, 'build')
   make_cpp_tests(ci_cpu, 'build')
-  {{ m.upload_artifacts(tag='cpu', filenames=tvm_multilib_tsim + tvm_allvisible + crttest + cpptest + standalone_crt) }}
+  {{ m.upload_artifacts(tag='cpu', filenames=tvm_multilib_tsim + tvm_allvisible + crttest + cpptest + standalone_crt + microtvm_template_projects) }}
   ci_setup(ci_cpu)
   // sh "${docker_run} ${ci_cpu} ./tests/scripts/task_golang.sh"
   // TODO(@jroesch): need to resolve CI issue will turn back on in follow up patch
diff --git a/ci/jenkins/templates/i386_jenkinsfile.groovy.j2 b/ci/jenkins/templates/i386_jenkinsfile.groovy.j2
@@ -31,7 +31,7 @@
   cmake_build(ci_i386, 'build', '-j2')
   make_standalone_crt(ci_i386, 'build')
   make_cpp_tests(ci_i386, 'build')
-  {{ m.upload_artifacts(tag='i386', filenames=tvm_multilib_tsim + standalone_crt + crttest + cpptest) }}
+  {{ m.upload_artifacts(tag='i386', filenames=tvm_multilib_tsim + standalone_crt + crttest + cpptest + microtvm_template_projects) }}
 {% endcall %}
 
 
diff --git a/ci/jenkins/templates/minimal_jenkinsfile.groovy.j2 b/ci/jenkins/templates/minimal_jenkinsfile.groovy.j2
@@ -31,7 +31,7 @@
   cmake_build(ci_minimal, 'build', '-j2')
   make_standalone_crt(ci_minimal, 'build')
   make_cpp_tests(ci_minimal, 'build')
-  {{ m.upload_artifacts(tag='cpu-minimal', filenames=tvm_lib + tvm_allvisible + crttest + cpptest + standalone_crt) }}
+  {{ m.upload_artifacts(tag='cpu-minimal', filenames=tvm_lib + tvm_allvisible + crttest + cpptest + standalone_crt + microtvm_template_projects) }}
 {% endcall %}
 
 
diff --git a/ci/scripts/jenkins/s3.py b/ci/scripts/jenkins/s3.py
@@ -142,4 +142,7 @@ def s3(source: str, destination: str, recursive: bool) -> List[str]:
                 show_md5(file)
         elif action == Action.UPLOAD:
             show_md5(item)
+            if Path(item).is_dir():
+                if len(list(Path(item).glob("**/*"))) == 0:
+                    raise RuntimeError(f"Cannot upload empty folder with name: {item}")
             s3(item, s3_path + "/" + item, recursive=Path(item).is_dir())
diff --git a/cmake/modules/CRT.cmake b/cmake/modules/CRT.cmake
@@ -0,0 +1,74 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more contributor
+# license agreements.  See the NOTICE file distributed with this work for
+# additional information regarding copyright ownership.  The ASF licenses this
+# file to you under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License.  You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+if(USE_MICRO)
+  message(STATUS "Add CRT template project for microTVM")
+
+  function(microtvm_add_crt)
+    list(
+      APPEND
+      CRT_TEMPLATE_FILE_COPY_JOBS
+      "src/runtime/crt/host microtvm_api_server.py -> crt"
+      "src/runtime/crt/host Makefile.template -> crt"
+      "src/runtime/crt crt_config-template.h -> crt"
+      "src/runtime/crt/host main.cc -> crt/src"
+    )
+
+    foreach(job_spec IN LISTS CRT_TEMPLATE_FILE_COPY_JOBS)
+      string(REPLACE " " ";" job_spec "${job_spec}")
+      list(LENGTH job_spec job_spec_length)
+      math(EXPR job_spec_length_mod "${job_spec_length} % 3")
+      if(NOT "${job_spec_length_mod}" EQUAL 1)
+        message(
+          FATAL_ERROR
+            "CRT copy job spec list length is ${job_spec_length}; parsed job spec is ${job_spec}"
+        )
+      endif()
+      math(EXPR job_spec_stop "${job_spec_length} - 3")
+
+      list(GET job_spec 0 job_src_base)
+      set(job_src_base "${CMAKE_CURRENT_SOURCE_DIR}/${job_src_base}")
+      foreach(copy_pattern_index RANGE 1 "${job_spec_stop}" 3)
+        list(GET job_spec ${copy_pattern_index} copy_pattern)
+        math(EXPR copy_dest_index "${copy_pattern_index} + 2")
+        list(GET job_spec ${copy_dest_index} copy_dest)
+
+        file(
+          GLOB_RECURSE copy_files
+          RELATIVE "${job_src_base}"
+          "${job_src_base}/${copy_pattern}")
+        list(LENGTH copy_files copy_files_length)
+        if("${copy_files_length}" EQUAL 0)
+          message(
+            FATAL_ERROR
+              "CRT copy job matched 0 files: ${job_src_base}/${copy_pattern} -> ${copy_dest}"
+          )
+        endif()
+        foreach(copy_src IN LISTS copy_files)
+          get_filename_component(
+            dest_path "${MICROTVM_TEMPLATE_PROJECTS}/${copy_dest}/${copy_src}"
+            ABSOLUTE)
+          tvm_micro_add_copy_file(crt_template_deps
+                                  ${job_src_base}/${copy_src} ${dest_path})
+        endforeach()
+      endforeach()
+    endforeach()
+
+    add_custom_target(crt DEPENDS ${crt_template_deps})
+  endfunction()
+
+  microtvm_add_crt()
+
+endif(USE_MICRO)
diff --git a/cmake/modules/StandaloneCrt.cmake b/cmake/modules/StandaloneCrt.cmake
@@ -94,16 +94,12 @@ else()
         "src/runtime/crt/common *.c -> src/runtime/crt/common"
         "src/runtime/crt/graph_executor *.c -> src/runtime/crt/graph_executor"
         "src/runtime/crt/graph_executor_module *.c -> src/runtime/crt/graph_executor_module"
-        "src/runtime/crt/host *.cc -> template/host"
-        "src/runtime/crt/host *.py -> template/host"
-        "src/runtime/crt/host Makefile.template -> template/host"
         "src/runtime/crt/memory *.c -> src/runtime/crt/memory"
         "src/runtime/crt/microtvm_rpc_common *.cc -> src/runtime/crt/microtvm_rpc_common"
         "src/runtime/crt/microtvm_rpc_server *.cc -> src/runtime/crt/microtvm_rpc_server"
         "src/runtime/minrpc *.h -> src/runtime/minrpc"
         "src/support generic_arena.h -> src/support"
         "src/support ssize.h -> src/support"
-        "src/runtime/crt crt_config-template.h -> template"
         )
 
   set(STANDALONE_CRT_BASE ${CMAKE_CURRENT_BINARY_DIR}/standalone_crt)
diff --git a/python/tvm/micro/build.py b/python/tvm/micro/build.py
@@ -94,9 +94,6 @@ def get_microtvm_template_projects(platform: str) -> str:
     if platform not in MicroTVMTemplateProject.list():
         raise ValueError(f"platform {platform} is not supported.")
 
-    if platform == MicroTVMTemplateProject.CRT.value:
-        return os.path.join(get_standalone_crt_dir(), "template", "host")
-
     microtvm_template_projects = None
     for path in libinfo.find_lib_path():
         template_path = os.path.join(os.path.dirname(path), "microtvm_template_projects")
diff --git a/python/tvm/testing/aot.py b/python/tvm/testing/aot.py
@@ -723,9 +723,9 @@ def run_and_check_body(base_path):
 
         include_path = os.path.join(base_path, "include")
         os.mkdir(include_path)
-        crt_root = tvm.micro.get_standalone_crt_dir()
+        crt_root = tvm.micro.get_microtvm_template_projects("crt")
         shutil.copy2(
-            os.path.join(crt_root, "template", "crt_config-template.h"),
+            os.path.join(crt_root, "crt_config-template.h"),
             os.path.join(include_path, "crt_config.h"),
         )
 
diff --git a/src/runtime/crt/host/microtvm_api_server.py b/src/runtime/crt/host/microtvm_api_server.py
@@ -144,15 +144,16 @@ def generate_project(self, model_library_format_path, standalone_crt_dir, projec
         crt_config_dir = project_dir / "crt_config"
         crt_config_dir.mkdir()
         shutil.copy2(
-            os.path.join(os.path.dirname(__file__), "..", "crt_config-template.h"),
+            os.path.join(os.path.dirname(__file__), "crt_config-template.h"),
             os.path.join(crt_config_dir, "crt_config.h"),
         )
 
         # Populate src/
         src_dir = os.path.join(project_dir, "src")
         os.mkdir(src_dir)
         shutil.copy2(
-            os.path.join(os.path.dirname(__file__), "main.cc"), os.path.join(src_dir, "main.cc")
+            os.path.join(os.path.dirname(__file__), "src", "main.cc"),
+            os.path.join(src_dir, "main.cc"),
         )
 
     def build(self, options):
diff --git a/tests/python/unittest/test_crt.py b/tests/python/unittest/test_crt.py
@@ -47,7 +47,7 @@ def _make_sess_from_op(temp_dir, op_name, sched, arg_bufs):
 
 
 def _make_session(temp_dir, mod):
-    template_project_dir = os.path.join(tvm.micro.get_standalone_crt_dir(), "template", "host")
+    template_project_dir = pathlib.Path(tvm.micro.get_microtvm_template_projects("crt"))
     project = tvm.micro.generate_project(
         template_project_dir, mod, temp_dir / "project", {"verbose": 1}
     )
diff --git a/tests/python/unittest/test_link_params.py b/tests/python/unittest/test_link_params.py
@@ -345,7 +345,7 @@ def test_crt_link_params(linkable_dtype):
         assert len(factory.get_params().keys()) == 0  # NOTE: params became tir.constants
 
         temp_dir = tvm.contrib.utils.tempdir()
-        template_project_dir = os.path.join(tvm.micro.get_standalone_crt_dir(), "template", "host")
+        template_project_dir = tvm.micro.get_microtvm_template_projects("crt")
         project = tvm.micro.generate_project(
             template_project_dir, factory, temp_dir / "project", {"verbose": 1}
         )
