[CI][Hexagon] Add Hexagon Tests to pipeline

Jenkinsfile

@@ -52,6 +52,7 @@ ci_wasm = "tlcpack/ci-wasm:v0.71"
 ci_i386 = "tlcpack/ci-i386:v0.74"
 ci_qemu = "tlcpack/ci-qemu:v0.10"
 ci_arm = "tlcpack/ci-arm:v0.07"
+ci_hexagon = "tlcpack/ci-hexagon:v0.01"
 // <--- End of regex-scanned config.
 
 // Parameters to allow overriding (in Jenkins UI), the images
@@ -381,6 +382,34 @@ stage('Build') {
      } else {
       Utils.markStageSkippedForConditional('BUILD: QEMU')
     }
+  },
+  'BUILD: Hexagon': {
+    if (!skip_ci && is_docs_only_build != 1) {
+      node('CPU') {
+        ws(per_exec_ws('tvm/build-hexagon')) {
+          init_git()
+          sh (
+            script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_config_build_hexagon.sh",
+            label: 'Create Hexagon cmake config',
+          )
+          try {
+            make(ci_hexagon, 'build', '-j2')
+            sh (
+              script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_build_hexagon_api.sh",
+              label: 'Build Hexagon API',
+            )
+            sh (
+              script: "${docker_run} ${ci_qemu} ./tests/scripts/task_python_hexagon.sh",
+              label: 'Run Hexagon tests',
+            )
+          } finally {
+            junit 'build/pytest-results/*.xml'
+          }
+        }
+      }
+     } else {
+      Utils.markStageSkippedForConditional('BUILD: Hexagon')
+    }
   }
 }
 

tests/python/contrib/test_hexagon/conftest.py

@@ -27,8 +27,8 @@
 HEXAGON_TOOLCHAIN = "HEXAGON_TOOLCHAIN"
 TVM_TRACKER_HOST = "TVM_TRACKER_HOST"
 TVM_TRACKER_PORT = "TVM_TRACKER_PORT"
-ANDROID_TRACKER_KEY = "ANDROID_TRACKER_KEY"
 ANDROID_REMOTE_DIR = "ANDROID_REMOTE_DIR"
+ANDROID_SERIAL_NUMBER = "ANDROID_SERIAL_NUMBER"
 
 
 @tvm.testing.fixture
@@ -49,83 +49,26 @@ def _compose(args, decs):
 def requires_hexagon_toolchain(*args):
     _requires_hexagon_toolchain = [
         pytest.mark.skipif(
-            os.environ.get("HEXAGON_TOOLCHAIN") == None,
-            reason="HEXAGON_TOOLCHAIN environment variable is required to run this test.",
+            os.environ.get(HEXAGON_TOOLCHAIN) == None,
+            reason=f"Missing environment variable, {HEXAGON_TOOLCHAIN}.",
         ),
     ]
 
     return _compose(args, _requires_hexagon_toolchain)
 
 
 @tvm.testing.fixture
-def android_tracker_key():
-    return os.environ["ANDROID_TRACKER_KEY"]
+def android_serial_number():
+    return os.getenv(ANDROID_SERIAL_NUMBER, default=None)
 
 
 @tvm.testing.fixture
 def tvm_tracker_host():
-    return os.environ["TVM_TRACKER_HOST"]
+    return os.getenv(TVM_TRACKER_HOST, default=None)
 
 
 @tvm.testing.fixture
 def tvm_tracker_port():
-    return int(os.environ["TVM_TRACKER_PORT"])
-
-
-@tvm.testing.fixture
-def remote_path():
-    dso_binary = "test_binary.so"
-    return os.path.join(os.environ["ANDROID_REMOTE_DIR"], dso_binary)
-
-
-@tvm.testing.fixture
-def rpc_sess(android_tracker_key, tvm_tracker_host, tvm_tracker_port):
-    from tvm import rpc
-
-    tracker = rpc.connect_tracker(tvm_tracker_host, tvm_tracker_port)
-    remote = tracker.request(android_tracker_key, priority=0, session_timeout=600)
-    return remote
-
-
-def requires_rpc_tracker_and_android_key(*args):
-    """Mark a test as requiring an RPC tracker to exist in
-    the host environment to run."""
-    _requires_rpc_tracker = [
-        *tvm.testing.requires_rpc(),
-        pytest.mark.skipif(
-            os.environ.get(TVM_TRACKER_HOST) == None,
-            reason="Missing environment variable, TVM_TRACKER_HOST",
-        ),
-        pytest.mark.skipif(
-            os.environ.get(TVM_TRACKER_PORT) == None,
-            reason="Missing environment variable, TVM_TRACKER_PORT",
-        ),
-        pytest.mark.skipif(
-            os.environ.get(ANDROID_TRACKER_KEY) == None,
-            reason="Missing environment variable, ANDROID_TRACKER_KEY",
-        ),
-        pytest.mark.skipif(
-            os.environ.get(ANDROID_REMOTE_DIR) == None,
-            reason="Missing environment variable, ANDROID_REMOTE_DIR",
-        ),
-    ]
-
-    return _compose(args, _requires_rpc_tracker)
-
-
-def requires_rpc_tracker(*args):
-    """Mark a test as requiring an RPC tracker to exist in
-    the host environment to run."""
-    _requires_rpc_tracker = [
-        *tvm.testing.requires_rpc(),
-        pytest.mark.skipif(
-            os.environ.get("TVM_TRACKER_HOST") == None,
-            reason="Missing environment variable, TVM_TRACKER_HOST",
-        ),
-        pytest.mark.skipif(
-            os.environ.get("TVM_TRACKER_PORT") == None,
-            reason="Missing environment variable, TVM_TRACKER_PORT",
-        ),
-    ]
-
-    return _compose(args, _requires_rpc_tracker)
+    port = os.getenv(TVM_TRACKER_PORT, default=None)
+    port = int(port) if port else None
+    return port

tests/python/contrib/test_hexagon/rpc/test_launcher.py → tests/python/contrib/test_hexagon/test_launcher.py

@@ -18,7 +18,7 @@
 import sys
 import pytest
 import numpy as np
-import os
+import logging
 
 import tvm.testing
 from tvm import te
@@ -28,12 +28,11 @@
 from tvm.contrib.hexagon.build import HexagonLauncher
 import tvm.contrib.hexagon.hexagon as hexagon
 
-from ..conftest import requires_rpc_tracker, requires_hexagon_toolchain
+from .conftest import requires_hexagon_toolchain
 
 
-@requires_rpc_tracker
 @requires_hexagon_toolchain
-def test_add(tvm_tracker_host, tvm_tracker_port, android_serial_number):
+def test_add(android_serial_number, tvm_tracker_host, tvm_tracker_port):
     dtype = "int8"
     A = tvm.te.placeholder((2,), dtype=dtype)
     B = tvm.te.placeholder((1,), dtype=dtype)
@@ -50,6 +49,10 @@ def test_add(tvm_tracker_host, tvm_tracker_port, android_serial_number):
     dso_binary_path = temp.relpath(dso_binary)
     func.save(dso_binary_path)
 
+    if not android_serial_number:
+        logging.info("Skip hardware test.")
+        return
+
     launcher = HexagonLauncher(serial_number=android_serial_number)
     launcher.android_run_rpc(rpc_tracker_host=tvm_tracker_host, rpc_tracker_port=tvm_tracker_port)
     launcher.hexagon_setup()
@@ -76,9 +79,8 @@ def test_add(tvm_tracker_host, tvm_tracker_port, android_serial_number):
     launcher.close()
 
 
-@requires_rpc_tracker
 @requires_hexagon_toolchain
-def test_add_vtcm(tvm_tracker_host, tvm_tracker_port, android_serial_number):
+def test_add_vtcm(android_serial_number, tvm_tracker_host, tvm_tracker_port):
     dtype = "int8"
     A = tvm.te.placeholder((2,), dtype=dtype)
     B = tvm.te.placeholder((1,), dtype=dtype)
@@ -95,6 +97,10 @@ def test_add_vtcm(tvm_tracker_host, tvm_tracker_port, android_serial_number):
     dso_binary_path = temp.relpath(dso_binary)
     func.save(dso_binary_path)
 
+    if not android_serial_number:
+        logging.info("Skip hardware test.")
+        return
+
     launcher = HexagonLauncher(serial_number=android_serial_number)
     launcher.android_run_rpc(rpc_tracker_host=tvm_tracker_host, rpc_tracker_port=tvm_tracker_port)
     launcher.hexagon_setup()
@@ -129,9 +135,8 @@ class TestMatMul:
     N = tvm.testing.parameter(32)
     K = tvm.testing.parameter(32)
 
-    @requires_rpc_tracker
     @requires_hexagon_toolchain
-    def test_matmul(self, tvm_tracker_host, tvm_tracker_port, android_serial_number, M, N, K):
+    def test_matmul(self, android_serial_number, tvm_tracker_host, tvm_tracker_port, M, N, K):
         X = te.placeholder((M, K), dtype="float32")
         Y = te.placeholder((K, N), dtype="float32")
         k1 = te.reduce_axis((0, K), name="k1")
@@ -148,6 +153,10 @@ def test_matmul(self, tvm_tracker_host, tvm_tracker_port, android_serial_number,
         dso_binary_path = temp.relpath(dso_binary)
         func.save(dso_binary_path)
 
+        if not android_serial_number:
+            logging.info("Skip hardware test.")
+            return
+
         launcher = HexagonLauncher(serial_number=android_serial_number)
         launcher.android_run_rpc(
             rpc_tracker_host=tvm_tracker_host, rpc_tracker_port=tvm_tracker_port
@@ -185,9 +194,8 @@ def test_matmul(self, tvm_tracker_host, tvm_tracker_port, android_serial_number,
         tvm.testing.assert_allclose(zt.numpy(), ztcpu.numpy(), rtol=1e-4)
 
 
-@requires_rpc_tracker
 @requires_hexagon_toolchain
-def test_graph_executor(tvm_tracker_host, tvm_tracker_port, android_serial_number):
+def test_graph_executor(android_serial_number, tvm_tracker_host, tvm_tracker_port):
     dtype = "float32"
     data = relay.var("data", relay.TensorType((1, 64, 64, 3), dtype))
     weight = relay.var("weight", relay.TensorType((5, 5, 3, 8), dtype))
@@ -221,6 +229,10 @@ def test_graph_executor(tvm_tracker_host, tvm_tracker_port, android_serial_numbe
         )
         lowered.get_lib().save(dso_binary_path)
 
+    if not android_serial_number:
+        logging.info("Skip hardware test.")
+        return
+
     launcher = HexagonLauncher(serial_number=android_serial_number)
     launcher.android_run_rpc(rpc_tracker_host=tvm_tracker_host, rpc_tracker_port=tvm_tracker_port)
     launcher.hexagon_setup()

tests/python/contrib/test_hexagon/rpc/__init__.py → tests/scripts/task_python_hexagon.sh

@@ -1,3 +1,4 @@
+#!/usr/bin/env bash
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -15,4 +16,12 @@
 # specific language governing permissions and limitations
 # under the License.
 
-""" Testing infrastructure for Hexagon RPC"""
+set -e
+set -u
+
+source tests/scripts/setup-pytest-env.sh
+
+make cython3
+
+unset ANDROID_SERIAL_NUMBER
+run_pytest ctypes python-contrib-hexagon-no-hwardware tests/python/contrib/test_hexagon/test_launcher.py