[CI][Hexagon] Add Hexagon Tests to pipeline (#10302)

* Add hexagon tests to CI Hexagon

* Fix CRT libs

* cleanup and fix Jenkins

* Address @areusch comments

Author: mehrdadh

Jenkinsfile

@@ -52,6 +52,7 @@ ci_wasm = "tlcpack/ci-wasm:v0.71"
 ci_i386 = "tlcpack/ci-i386:v0.74"
 ci_qemu = "tlcpack/ci-qemu:v0.10"
 ci_arm = "tlcpack/ci-arm:v0.07"
+ci_hexagon = "tlcpack/ci-hexagon:v0.01"
 // <--- End of regex-scanned config.
 
 // Parameters to allow overriding (in Jenkins UI), the images
@@ -381,6 +382,34 @@ stage('Build') {
      } else {
       Utils.markStageSkippedForConditional('BUILD: QEMU')
     }
+  },
+  'BUILD: Hexagon': {
+    if (!skip_ci && is_docs_only_build != 1) {
+      node('CPU') {
+        ws(per_exec_ws('tvm/build-hexagon')) {
+          init_git()
+          sh (
+            script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_config_build_hexagon.sh",
+            label: 'Create Hexagon cmake config',
+          )
+          try {
+            make(ci_hexagon, 'build', '-j2')
+            sh (
+              script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_build_hexagon_api.sh",
+              label: 'Build Hexagon API',
+            )
+            sh (
+              script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_python_hexagon.sh",
+              label: 'Run Hexagon tests',
+            )
+          } finally {
+            junit 'build/pytest-results/*.xml'
+          }
+        }
+      }
+     } else {
+      Utils.markStageSkippedForConditional('BUILD: Hexagon')
+    }
   }
 }
 

tests/python/contrib/test_hexagon/conftest.py

@@ -27,8 +27,8 @@
 HEXAGON_TOOLCHAIN = "HEXAGON_TOOLCHAIN"
 TVM_TRACKER_HOST = "TVM_TRACKER_HOST"
 TVM_TRACKER_PORT = "TVM_TRACKER_PORT"
-ANDROID_TRACKER_KEY = "ANDROID_TRACKER_KEY"
 ANDROID_REMOTE_DIR = "ANDROID_REMOTE_DIR"
+ANDROID_SERIAL_NUMBER = "ANDROID_SERIAL_NUMBER"
 
 
 @tvm.testing.fixture
@@ -49,83 +49,26 @@ def _compose(args, decs):
 def requires_hexagon_toolchain(*args):
     _requires_hexagon_toolchain = [
         pytest.mark.skipif(
-            os.environ.get("HEXAGON_TOOLCHAIN") == None,
-            reason="HEXAGON_TOOLCHAIN environment variable is required to run this test.",
+            os.environ.get(HEXAGON_TOOLCHAIN) == None,
+            reason=f"Missing environment variable {HEXAGON_TOOLCHAIN}.",
         ),
     ]
 
     return _compose(args, _requires_hexagon_toolchain)
 
 
 @tvm.testing.fixture
-def android_tracker_key():
-    return os.environ["ANDROID_TRACKER_KEY"]
+def android_serial_number():
+    return os.getenv(ANDROID_SERIAL_NUMBER, default=None)
 
 
 @tvm.testing.fixture
 def tvm_tracker_host():
-    return os.environ["TVM_TRACKER_HOST"]
+    return os.getenv(TVM_TRACKER_HOST, default=None)
 
 
 @tvm.testing.fixture
 def tvm_tracker_port():
-    return int(os.environ["TVM_TRACKER_PORT"])
-
-
-@tvm.testing.fixture
-def remote_path():
-    dso_binary = "test_binary.so"
-    return os.path.join(os.environ["ANDROID_REMOTE_DIR"], dso_binary)
-
-
-@tvm.testing.fixture
-def rpc_sess(android_tracker_key, tvm_tracker_host, tvm_tracker_port):
-    from tvm import rpc
-
-    tracker = rpc.connect_tracker(tvm_tracker_host, tvm_tracker_port)
-    remote = tracker.request(android_tracker_key, priority=0, session_timeout=600)
-    return remote
-
-
-def requires_rpc_tracker_and_android_key(*args):
-    """Mark a test as requiring an RPC tracker to exist in
-    the host environment to run."""
-    _requires_rpc_tracker = [
-        *tvm.testing.requires_rpc(),
-        pytest.mark.skipif(
-            os.environ.get(TVM_TRACKER_HOST) == None,
-            reason="Missing environment variable, TVM_TRACKER_HOST",
-        ),
-        pytest.mark.skipif(
-            os.environ.get(TVM_TRACKER_PORT) == None,
-            reason="Missing environment variable, TVM_TRACKER_PORT",
-        ),
-        pytest.mark.skipif(
-            os.environ.get(ANDROID_TRACKER_KEY) == None,
-            reason="Missing environment variable, ANDROID_TRACKER_KEY",
-        ),
-        pytest.mark.skipif(
-            os.environ.get(ANDROID_REMOTE_DIR) == None,
-            reason="Missing environment variable, ANDROID_REMOTE_DIR",
-        ),
-    ]
-
-    return _compose(args, _requires_rpc_tracker)
-
-
-def requires_rpc_tracker(*args):
-    """Mark a test as requiring an RPC tracker to exist in
-    the host environment to run."""
-    _requires_rpc_tracker = [
-        *tvm.testing.requires_rpc(),
-        pytest.mark.skipif(
-            os.environ.get("TVM_TRACKER_HOST") == None,
-            reason="Missing environment variable, TVM_TRACKER_HOST",
-        ),
-        pytest.mark.skipif(
-            os.environ.get("TVM_TRACKER_PORT") == None,
-            reason="Missing environment variable, TVM_TRACKER_PORT",
-        ),
-    ]
-
-    return _compose(args, _requires_rpc_tracker)
+    port = os.getenv(TVM_TRACKER_PORT, default=None)
+    port = int(port) if port else None
+    return port

tests/python/contrib/test_hexagon/rpc/conftest.py

@@ -19,12 +19,12 @@
 HexagonLauncher is a class to handle interactions with an Android phone which includes Hexagon DSP to run a TVMModule(function/operation/graph) on Hexagon. HexagonLauncher reuses minRPC implementation to setup an RPC connection from host (your local machine) to Hexagon target which is passed through Android RPC server.
 
 ## Build Required Tools/Libraries
-Here are the steps that are taken to prepare a runtime on a Hexagon device to test any model.
+To build TVM for Hexagon and run tests you can follow these steps to prepare a runtime on a Hexagon device to test any model. Alternatively, you can skip these instructions and use docker image which has pre-installed required tools. Instructions for using docker image [here](#use-hexagon-docker-image).
 
 - Build TVMRuntime library and C++ RPC server for Android.
 - Build minRPC server along with FastRPC for Hexagon.
 - Build TVM library with Hexagon support for host machine.
-- Build TVMRuntime library and C++ RPC server for host machine.
+- Build TVMRuntime library and RPC server for host machine.
 
 Note: First, ensure to export Clang libraries to `LD_LIBRARY_PATH` and Hexagon toolchain to `HEXAGON_TOOLCHAIN`:
 
@@ -58,14 +58,33 @@ cd tvm
 mkdir build
 cd build
 cmake -DUSE_LLVM="path to `llvm/bin/llvm-config`" \
-        -DUSE_CPP_RPC=ON \
+        -DUSE_RPC=ON \
         -DCMAKE_CXX_COMPILER="path to `clang++` executable" \
         -DCMAKE_CXX_FLAGS='-stdlib=libc++' \
         -DUSE_HEXAGON_SDK="path to Hexagon SDK" \
         -DUSE_HEXAGON_ARCH="choose from v65|v66|v68" \
         -DUSE_HEXAGON_DEVICE=sim ..
 ```
 
+## Use Hexagon Docker Image
+To use this docker image, install TVM and tools follow these steps.
+
+```bash
+# Log in to docker image
+cd tvm
+./docker/bash.sh tlcpack/ci-hexagon:v0.01
+
+# Build TVM
+./tests/scripts/task_config_build_hexagon.sh 
+cd build
+cmake ..
+make -j2
+
+# Build Hexagon API
+cd ..
+./tests/scripts/task_build_hexagon_api.sh 
+```
+
 ## Testing Using HexagonLauncher
 Before starting a test you need to run an RPC tracker on your local machine and export HOST and PORT as environment variables. Also, you need to export Clang libraries to `LD_LIBRARY_PATH` and Hexagon toolchain to `HEXAGON_TOOLCHAIN` as explained above.
 

tests/python/contrib/test_hexagon/rpc/test_launcher.md renamed to tests/python/contrib/test_hexagon/test_launcher.md

@@ -18,7 +18,7 @@
 import sys
 import pytest
 import numpy as np
-import os
+import logging
 
 import tvm.testing
 from tvm import te
@@ -28,12 +28,11 @@
 from tvm.contrib.hexagon.build import HexagonLauncher
 import tvm.contrib.hexagon.hexagon as hexagon
 
-from ..conftest import requires_rpc_tracker, requires_hexagon_toolchain
+from .conftest import requires_hexagon_toolchain
 
 
-@requires_rpc_tracker
 @requires_hexagon_toolchain
-def test_add(tvm_tracker_host, tvm_tracker_port, android_serial_number):
+def test_add(android_serial_number, tvm_tracker_host, tvm_tracker_port):
     dtype = "int8"
     A = tvm.te.placeholder((2,), dtype=dtype)
     B = tvm.te.placeholder((1,), dtype=dtype)
@@ -50,6 +49,9 @@ def test_add(tvm_tracker_host, tvm_tracker_port, android_serial_number):
     dso_binary_path = temp.relpath(dso_binary)
     func.save(dso_binary_path)
 
+    if not android_serial_number:
+        pytest.skip("Skip hardware test since ANDROID_SERIAL_NUMBER is not set.")
+
     launcher = HexagonLauncher(serial_number=android_serial_number)
     launcher.android_run_rpc(rpc_tracker_host=tvm_tracker_host, rpc_tracker_port=tvm_tracker_port)
     launcher.hexagon_setup()
@@ -76,9 +78,8 @@ def test_add(tvm_tracker_host, tvm_tracker_port, android_serial_number):
     launcher.close()
 
 
-@requires_rpc_tracker
 @requires_hexagon_toolchain
-def test_add_vtcm(tvm_tracker_host, tvm_tracker_port, android_serial_number):
+def test_add_vtcm(android_serial_number, tvm_tracker_host, tvm_tracker_port):
     dtype = "int8"
     A = tvm.te.placeholder((2,), dtype=dtype)
     B = tvm.te.placeholder((1,), dtype=dtype)
@@ -95,6 +96,9 @@ def test_add_vtcm(tvm_tracker_host, tvm_tracker_port, android_serial_number):
     dso_binary_path = temp.relpath(dso_binary)
     func.save(dso_binary_path)
 
+    if not android_serial_number:
+        pytest.skip("Skip hardware test since ANDROID_SERIAL_NUMBER is not set.")
+
     launcher = HexagonLauncher(serial_number=android_serial_number)
     launcher.android_run_rpc(rpc_tracker_host=tvm_tracker_host, rpc_tracker_port=tvm_tracker_port)
     launcher.hexagon_setup()
@@ -129,9 +133,8 @@ class TestMatMul:
     N = tvm.testing.parameter(32)
     K = tvm.testing.parameter(32)
 
-    @requires_rpc_tracker
     @requires_hexagon_toolchain
-    def test_matmul(self, tvm_tracker_host, tvm_tracker_port, android_serial_number, M, N, K):
+    def test_matmul(self, android_serial_number, tvm_tracker_host, tvm_tracker_port, M, N, K):
         X = te.placeholder((M, K), dtype="float32")
         Y = te.placeholder((K, N), dtype="float32")
         k1 = te.reduce_axis((0, K), name="k1")
@@ -148,6 +151,9 @@ def test_matmul(self, tvm_tracker_host, tvm_tracker_port, android_serial_number,
         dso_binary_path = temp.relpath(dso_binary)
         func.save(dso_binary_path)
 
+        if not android_serial_number:
+            pytest.skip("Skip hardware test since ANDROID_SERIAL_NUMBER is not set.")
+
         launcher = HexagonLauncher(serial_number=android_serial_number)
         launcher.android_run_rpc(
             rpc_tracker_host=tvm_tracker_host, rpc_tracker_port=tvm_tracker_port
@@ -185,9 +191,8 @@ def test_matmul(self, tvm_tracker_host, tvm_tracker_port, android_serial_number,
         tvm.testing.assert_allclose(zt.numpy(), ztcpu.numpy(), rtol=1e-4)
 
 
-@requires_rpc_tracker
 @requires_hexagon_toolchain
-def test_graph_executor(tvm_tracker_host, tvm_tracker_port, android_serial_number):
+def test_graph_executor(android_serial_number, tvm_tracker_host, tvm_tracker_port):
     dtype = "float32"
     data = relay.var("data", relay.TensorType((1, 64, 64, 3), dtype))
     weight = relay.var("weight", relay.TensorType((5, 5, 3, 8), dtype))
@@ -221,6 +226,9 @@ def test_graph_executor(tvm_tracker_host, tvm_tracker_port, android_serial_numbe
         )
         lowered.get_lib().save(dso_binary_path)
 
+    if not android_serial_number:
+        pytest.skip("Skip hardware test since ANDROID_SERIAL_NUMBER is not set.")
+
     launcher = HexagonLauncher(serial_number=android_serial_number)
     launcher.android_run_rpc(rpc_tracker_host=tvm_tracker_host, rpc_tracker_port=tvm_tracker_port)
     launcher.hexagon_setup()

tests/python/contrib/test_hexagon/rpc/test_launcher.py renamed to tests/python/contrib/test_hexagon/test_launcher.py

@@ -23,9 +23,14 @@ mkdir -p build
 cd build
 cp ../cmake/config.cmake .
 
+echo set\(USE_SORT ON\) >> config.cmake
+echo set\(USE_RPC ON\) >> config.cmake
+echo set\(USE_MICRO ON\) >> config.cmake
+echo set\(USE_MICRO_STANDALONE_RUNTIME ON\) >> config.cmake
 echo set\(USE_LLVM "${CLANG_LLVM_HOME}/bin/llvm-config"\) >> config.cmake
-echo set\(USE_CPP_RPC ON\) >> config.cmake
 echo set\(CMAKE_CXX_COMPILER "${CLANG_LLVM_HOME}/bin/clang++"\) >> config.cmake
 echo set\(USE_HEXAGON_SDK "${HEXAGON_SDK_PATH}"\) >> config.cmake
 echo set\(USE_HEXAGON_ARCH "v68"\) >> config.cmake
 echo set\(USE_HEXAGON_DEVICE "sim"\) >> config.cmake
+echo set\(USE_CCACHE OFF\) >> config.cmake
+echo set\(SUMMARIZE ON\) >> config.cmake

tests/scripts/task_config_build_hexagon.sh

@@ -1,3 +1,4 @@
+#!/usr/bin/env bash
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -15,4 +16,13 @@
 # specific language governing permissions and limitations
 # under the License.
 
-""" Testing infrastructure for Hexagon RPC"""
+set -e
+set -u
+
+source tests/scripts/setup-pytest-env.sh
+
+make cython3
+
+# unset because hardware does not exist in CI.
+unset ANDROID_SERIAL_NUMBER
+run_pytest ctypes python-contrib-hexagon-no-hwardware tests/python/contrib/test_hexagon/test_launcher.py