[Hexagon] capture gtest output and return over FFI (#11239)

* [Hexagon] capture gtest output and return over FFI

* rename to test_hexagon_unit_tests.py so it will run in CI

* rename to run_unit_tests.cc

* pass back gtest error code along with gtest output

* skip Hexagon unit tests if gtest not enabled

* pass gtest_args as pytest argument

* change env variable to HEXAGON_GTEST

* set HEXAGON_GTEST in the environment to enable Hexagon unit tests

* add back try / except around get_function

Author: adstraw

docker/Dockerfile.ci_hexagon

@@ -63,6 +63,7 @@ ENV CLANG_LLVM_HOME /opt/clang-llvm
 ENV LD_LIBRARY_PATH $LD_LIBRARY_PATH:/opt/clang-llvm/lib
 ENV PATH /opt/clang-llvm/bin:$PATH
 ENV HEXAGON_TOOLCHAIN "${HEXAGON_SDK_PATH}/tools/HEXAGON_Tools/8.5.08/Tools"
+ENV HEXAGON_GTEST "${HEXAGON_SDK_PATH}/utils/googletest/gtest"
 
 # sccache
 COPY install/ubuntu_install_sccache.sh /install/ubuntu_install_sccache.sh

tests/cpp-runtime/hexagon/run_unit_tests.cc

@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <gtest/gtest.h>
+#include <tvm/runtime/packed_func.h>
+#include <tvm/runtime/registry.h>
+
+#include <string>
+#include <vector>
+
+#include "../src/support/utils.h"
+
+namespace tvm {
+namespace runtime {
+namespace hexagon {
+
+class GtestPrinter : public testing::EmptyTestEventListener {
+  void OnTestProgramStart(const testing::UnitTest& unit_test) override {
+    gtest_out_ << "[==========] Running " << unit_test.test_to_run_count() << " test(s) from "
+               << unit_test.test_suite_to_run_count() << " test suite(s).\n";
+  }
+
+  void OnTestProgramEnd(const testing::UnitTest& unit_test) override {
+    gtest_out_ << "[==========] " << unit_test.test_to_run_count() << " test(s) from "
+               << unit_test.test_suite_to_run_count() << " test suite(s) ran. ("
+               << unit_test.elapsed_time() << " ms total)\n";
+    gtest_out_ << "[  PASSED  ] " << unit_test.successful_test_count() << " test(s)\n";
+
+    if (unit_test.failed_test_count()) {
+      gtest_out_ << "[  FAILED  ] " << unit_test.failed_test_count() << " test(s)\n";
+    }
+  }
+
+  void OnTestSuiteStart(const testing::TestSuite& test_suite) override {
+    gtest_out_ << "[----------] " << test_suite.test_to_run_count() << " test(s) from "
+               << test_suite.name() << "\n";
+  }
+
+  void OnTestSuiteEnd(const testing::TestSuite& test_suite) override {
+    gtest_out_ << "[----------] " << test_suite.test_to_run_count() << " test(s) from "
+               << test_suite.name() << " (" << test_suite.elapsed_time() << " ms total)\n";
+  }
+
+  void OnTestStart(const testing::TestInfo& test_info) override {
+    gtest_out_ << "[ RUN      ] " << test_info.test_suite_name() << "." << test_info.name() << "\n";
+  }
+
+  void OnTestEnd(const testing::TestInfo& test_info) override {
+    for (int i = 0; i < test_info.result()->total_part_count(); ++i) {
+      gtest_out_ << test_info.result()->GetTestPartResult(i).message() << "\n";
+    }
+    if (test_info.result()->Passed()) {
+      gtest_out_ << "[       OK ]";
+    } else {
+      gtest_out_ << "[  FAILED  ]";
+    }
+    gtest_out_ << " " << test_info.test_suite_name() << "." << test_info.name() << " ("
+               << test_info.result()->elapsed_time() << " ms)\n";
+  }
+
+  std::stringstream gtest_out_;
+
+ public:
+  std::string GetOutput() { return gtest_out_.str(); }
+};
+
+TVM_REGISTER_GLOBAL("hexagon.run_unit_tests").set_body([](TVMArgs args, TVMRetValue* rv) {
+  // gtest args are passed into this packed func as a singular string
+  // split gtest args using <space> delimiter and build argument vector
+  std::vector<std::string> parsed_args = tvm::support::Split(args[0], ' ');
+  std::vector<char*> argv;
+
+  // add executable name
+  argv.push_back(const_cast<char*>("hexagon_run_unit_tests"));
+
+  // add parsed arguments
+  for (int i = 0; i < parsed_args.size(); ++i) {
+    argv.push_back(const_cast<char*>(parsed_args[i].data()));
+  }
+
+  // end of parsed arguments
+  argv.push_back(nullptr);
+
+  // set argument count
+  int argc = argv.size() - 1;
+
+  // initialize gtest with arguments and run
+  ::testing::InitGoogleTest(&argc, argv.data());
+
+  // add printer to capture gtest output in a string
+  GtestPrinter* gprinter = new GtestPrinter();
+  testing::TestEventListeners& listeners = testing::UnitTest::GetInstance()->listeners();
+  listeners.Append(gprinter);
+
+  int gtest_error_code = RUN_ALL_TESTS();
+  std::string gtest_output = gprinter->GetOutput();
+  std::stringstream gtest_error_code_and_output;
+  gtest_error_code_and_output << gtest_error_code << std::endl;
+  gtest_error_code_and_output << gtest_output;
+  *rv = gtest_error_code_and_output.str();
+  delete gprinter;
+});
+
+}  // namespace hexagon
+}  // namespace runtime
+}  // namespace tvm

tests/python/contrib/test_hexagon/conftest.py

@@ -218,3 +218,13 @@ def aot_target(aot_host_target):
         yield aot_host_target
     else:
         assert False, "Incorrect AoT host target: {aot_host_target}. Options are [c, llvm]."
+
+
+def pytest_addoption(parser):
+    parser.addoption("--gtest_args", action="store", default="")
+
+
+def pytest_generate_tests(metafunc):
+    option_value = metafunc.config.option.gtest_args
+    if "gtest_args" in metafunc.fixturenames and option_value is not None:
+        metafunc.parametrize("gtest_args", [option_value])

tests/python/contrib/test_hexagon/test_run_unit_tests.py

@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+import pytest
+import numpy as np
+from tvm.contrib.hexagon.build import HexagonLauncher
+from .conftest import requires_hexagon_toolchain
+
+
+# use pytest -sv to observe gtest output
+# use --gtest_args to pass arguments to gtest
+# for example to run all "foo" tests twice and observe gtest output run
+# pytest -sv <this file> --gtests_args="--gtest_filter=*foo* --gtest_repeat=2"
+@requires_hexagon_toolchain
+@pytest.mark.skipif(
+    os.environ.get("HEXAGON_GTEST") == None,
+    reason="Test requires environment variable HEXAGON_GTEST set with a path to a Hexagon gtest version normally located at /path/to/hexagon/sdk/utils/googletest/gtest",
+)
+def test_run_unit_tests(hexagon_session, gtest_args):
+    try:
+        func = hexagon_session._rpc.get_function("hexagon.run_unit_tests")
+    except:
+        print(
+            "Test requires TVM Runtime to be built with a Hexagon gtest version using Hexagon API cmake flag -DUSE_HEXAGON_GTEST=${HEXAGON_GTEST}"
+        )
+        raise
+
+    gtest_error_code_and_output = func(gtest_args)
+    gtest_error_code = int(gtest_error_code_and_output.splitlines()[0])
+    gtest_output = gtest_error_code_and_output.split("\n", 1)[-1]
+    print(gtest_output)
+    np.testing.assert_equal(gtest_error_code, 0)

tests/python/contrib/test_hexagon/unit_tests.py

@@ -37,13 +37,16 @@ cd build
 output_binary_directory=$(realpath ${PWD}/../../../build/hexagon_api_output)
 rm -rf ${output_binary_directory}
 
+# should be removed after Hexagon Docker update
+export HEXAGON_GTEST="${HEXAGON_SDK_PATH}/utils/googletest/gtest"
+
 cmake -DANDROID_ABI=arm64-v8a \
     -DANDROID_PLATFORM=android-28 \
     -DUSE_ANDROID_TOOLCHAIN="${ANDROID_NDK_HOME}/build/cmake/android.toolchain.cmake" \
     -DUSE_HEXAGON_ARCH=v68 \
     -DUSE_HEXAGON_SDK="${HEXAGON_SDK_PATH}" \
     -DUSE_HEXAGON_TOOLCHAIN="${HEXAGON_TOOLCHAIN}" \
     -DUSE_OUTPUT_BINARY_DIR="${output_binary_directory}" \
-    -DUSE_HEXAGON_GTEST="${HEXAGON_SDK_PATH}/utils/googletest/gtest" ..
+    -DUSE_HEXAGON_GTEST="${HEXAGON_GTEST}" ..
 
 make -j$(nproc)

tests/scripts/task_build_hexagon_api.sh

@@ -43,6 +43,9 @@ if [[ "${device_serial}" == "simulator" ]]; then
     export HEXAGON_SDK_ROOT=${HEXAGON_SDK_PATH}
 fi
 
+# should be removed after Hexagon Docker update
+export HEXAGON_GTEST="${HEXAGON_SDK_PATH}/utils/googletest/gtest"
+
 export ANDROID_SERIAL_NUMBER=${device_serial}
 run_pytest ctypes python-contrib-hexagon tests/python/contrib/test_hexagon