[Tizen] Run matter unit tests on QEMU (#26563)

* Run unit tests on Tizen platform under QEMU * Run all generated unit test executables * Move failing tests to XFAIL group * Manual for debugging tests on QEMU * Add "Run test (Linux x64)" to VSCode debugger * Print failed tests * Drop XFAIL since now all tests pass on Tizen * Do not duplicate example builds with QEMU tests * Do not add rootshell to the word list * Revert defines reordering * Add TODO for GN test assets --------- Co-authored-by: Andrei Litvin <[email protected]>
project-chip · Nov 22, 2023 · 7db34ac · 7db34ac
1 parent 6a2a76b
commit 7db34ac
Show file tree

Hide file tree

Showing 13 changed files with 197 additions and 36 deletions.
diff --git a/.github/workflows/examples-tizen.yaml b/.github/workflows/examples-tizen.yaml
@@ -25,7 +25,7 @@ concurrency:
 
 env:
     CHIP_NO_LOG_TIMESTAMPS: true
-    
+
 jobs:
     tizen:
         name: Tizen
@@ -54,10 +54,8 @@ jobs:
                       "./scripts/build/build_examples.py \
                           --enable-flashbundle \
                           --target tizen-arm-all-clusters \
-                          --target tizen-arm-all-clusters-minimal-no-wifi \
                           --target tizen-arm-chip-tool-ubsan \
                           --target tizen-arm-light \
-                          --target tizen-arm-light-no-ble-no-wifi \
                           build \
                           --copy-artifacts-to out/artifacts \
                       "
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -20,12 +20,20 @@
         },
         {
             "name": "Run application (Linux x64)",
-            "type": "lldb",
+            "type": "cppdbg",
             "request": "launch",
             "program": "${workspaceFolder}/out/${input:outAppLinux}",
             "args": [],
             "cwd": "${workspaceFolder}"
         },
+        {
+            "name": "Run test (Linux x64)",
+            "type": "cppdbg",
+            "request": "launch",
+            "program": "${workspaceFolder}/out/${input:outTestLinux}",
+            "args": [],
+            "cwd": "${workspaceFolder}"
+        },
         {
             "name": "QRCode Tests",
             "type": "cppdbg",
@@ -486,6 +494,15 @@
                 "description": "Select the application to run"
             }
         },
+        {
+            "type": "command",
+            "id": "outTestLinux",
+            "command": "shellCommand.execute",
+            "args": {
+                "command": "find ${workspaceFolder}/out/linux-x64-*/tests -type f -executable |sort |sed 's$${workspaceFolder}/out/$$'",
+                "description": "Select the test to run"
+            }
+        },
         {
             "type": "pickString",
             "id": "mbedDebugProfile",

diff --git a/build/chip/chip_test.gni b/build/chip/chip_test.gni
@@ -41,21 +41,23 @@ if (chip_link_tests) {
     group(_test_name + "_lib") {
     }
 
-    pw_python_action(_test_name + "_run") {
-      deps = [ ":${_test_name}" ]
-      inputs = [ pw_unit_test_AUTOMATIC_RUNNER ]
-      module = "pw_unit_test.test_runner"
-      python_deps = [
-        "$dir_pw_cli/py",
-        "$dir_pw_unit_test/py",
-      ]
-      args = [
-        "--runner",
-        rebase_path(pw_unit_test_AUTOMATIC_RUNNER, root_build_dir),
-        "--test",
-        rebase_path("$_test_output_dir/$_test_name", root_build_dir),
-      ]
-      stamp = true
+    if (chip_pw_run_tests) {
+      pw_python_action(_test_name + "_run") {
+        deps = [ ":${_test_name}" ]
+        inputs = [ pw_unit_test_AUTOMATIC_RUNNER ]
+        module = "pw_unit_test.test_runner"
+        python_deps = [
+          "$dir_pw_cli/py",
+          "$dir_pw_unit_test/py",
+        ]
+        args = [
+          "--runner",
+          rebase_path(pw_unit_test_AUTOMATIC_RUNNER, root_build_dir),
+          "--test",
+          rebase_path("$_test_output_dir/$_test_name", root_build_dir),
+        ]
+        stamp = true
+      }
     }
   }
 } else {

diff --git a/build/chip/chip_test_group.gni b/build/chip/chip_test_group.gni
@@ -60,11 +60,13 @@ template("chip_test_group") {
     }
   }
 
-  group("${_test_group_name}_run") {
-    if (chip_link_tests) {
-      deps = []
-      foreach(_test, invoker.deps) {
-        deps += [ get_label_info(_test, "label_no_toolchain") + "_run" ]
+  if (chip_pw_run_tests) {
+    group("${_test_group_name}_run") {
+      if (chip_link_tests) {
+        deps = []
+        foreach(_test, invoker.deps) {
+          deps += [ get_label_info(_test, "label_no_toolchain") + "_run" ]
+        }
       }
     }
   }

diff --git a/build/chip/chip_test_suite.gni b/build/chip/chip_test_suite.gni
@@ -151,10 +151,12 @@ template("chip_test_suite") {
       }
     }
 
-    group("${_suite_name}_run") {
-      deps = []
-      foreach(_test, tests) {
-        deps += [ ":${_test}_run" ]
+    if (chip_pw_run_tests) {
+      group("${_suite_name}_run") {
+        deps = []
+        foreach(_test, tests) {
+          deps += [ ":${_test}_run" ]
+        }
       }
     }
   } else {

diff --git a/build/chip/tests.gni b/build/chip/tests.gni
@@ -24,10 +24,17 @@ declare_args() {
 
 declare_args() {
   # Build executables for running individual tests.
-  chip_link_tests =
-      chip_build_tests && (current_os == "linux" || current_os == "mac") &&
-      current_cpu == target_cpu
+  chip_link_tests = chip_build_tests &&
+                    (current_os == "linux" || current_os == "mac" ||
+                     current_os == "tizen") && current_cpu == target_cpu
+}
 
+declare_args() {
+  # Run tests with pigweed test runner.
+  chip_pw_run_tests = chip_link_tests && current_os != "tizen"
+}
+
+declare_args() {
   # Use source_set instead of static_lib for tests.
   chip_build_test_static_libraries = chip_device_platform != "efr32"
 }

diff --git a/scripts/build/builders/tizen.py b/scripts/build/builders/tizen.py
@@ -113,6 +113,9 @@ def __init__(self,
             except FileNotFoundError:
                 pass
 
+        if app == TizenApp.TESTS:
+            self.extra_gn_options.append('chip_build_tests=true')
+
         if not enable_ble:
             self.extra_gn_options.append('chip_config_network_layer_ble=false')
         if not enable_thread:

diff --git a/src/test_driver/tizen/BUILD.gn b/src/test_driver/tizen/BUILD.gn
@@ -14,5 +14,8 @@
 
 group("check") {
   testonly = true
-  deps = [ "integration_tests/lighting-app:check" ]
+  deps = [
+    "chip_tests:check",
+    "integration_tests/lighting-app:check",
+  ]
 }
diff --git a/src/test_driver/tizen/README.md b/src/test_driver/tizen/README.md
@@ -55,3 +55,20 @@ gn gen --check --fail-on-unused-args \
 # Run Tizen QEMU-based tests
 ninja -C out/tizen-check check
 ```
+
+## Debugging Tests on QEMU
+
+When running tests with `ninja -C out/tizen-check check`, the test driver prints
+the command to run the test on QEMU. Please search for the following line in the
+output of the `ninja` command: `INFO:root:run: qemu-system-arm`. Make sure that
+your terminal does not limit the number of lines printed, or simply redirect the
+output to a file as follows:
+
+```sh
+ninja -C out/tizen-check check > ninja-tizen-check.log 2>&1
+```
+
+Then, use the run command and add the `rootshell` keyword to kernel arguments
+passed to QEMU (the string after the `-append` option). This will run QEMU, but
+instead of running the test, it will drop you to the shell. From there, you can
+run the test manually by typing `/mnt/chip/runner.sh`.
diff --git a/src/test_driver/tizen/chip_tests/BUILD.gn b/src/test_driver/tizen/chip_tests/BUILD.gn
@@ -0,0 +1,49 @@
+# Copyright (c) 2020 Project CHIP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import("//build_overrides/chip.gni")
+import("//build_overrides/tizen.gni")
+
+import("${tizen_sdk_build_root}/tizen_sdk.gni")
+
+tizen_qemu_mkisofs("chip-tests-runner") {
+  runner = "runner.sh"
+
+  # Build CHIP unit tests.
+  deps = [ "${chip_root}/src:tests" ]
+
+  # Bundle all created unit tests.
+  # TODO (arkq): Collect all unit tests in a way that we could add all targets
+  #              to the assets list. Right now QEMU runner dependencies are not
+  #              tracked properly. Changes to the unit tests will not trigger
+  #              rebuild of the ISO image, so the test will be run with old
+  #              binaries.
+  assets_non_tracked = [ rebase_path("${root_build_dir}/tests") ]
+}
+
+tizen_qemu_run("chip-tests") {
+  # Enable network support, so Tizen can obtain current date/time from the
+  # network. Correct date/time is required for the commissioning process -
+  # attestation will fail otherwise.
+  virtio_net = true
+
+  deps = [ ":chip-tests-runner" ]
+  mkisofs_outputs = get_target_outputs(":chip-tests-runner")
+  iso_image = rebase_path(mkisofs_outputs[0])
+}
+
+group("check") {
+  deps = [ ":chip-tests" ]
+  testonly = true
+}
diff --git a/src/test_driver/tizen/chip_tests/runner.sh b/src/test_driver/tizen/chip_tests/runner.sh
@@ -0,0 +1,53 @@
+#!/bin/bash
+
+#
+#    Copyright (c) 2021 Project CHIP Authors
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+set -e
+
+# Print CHIP logs on stdout
+dlogutil CHIP &
+
+FAILED=()
+STATUS=0
+
+# Run all executables in the /mnt/chip directory except the runner.sh script
+while IFS= read -r TEST; do
+
+    NAME=$(basename "$TEST")
+
+    echo
+    echo "RUN: $NAME"
+
+    RV=0
+    "$TEST" || RV=$?
+
+    if [ "$RV" -eq 0 ]; then
+        echo -e "DONE: \e[32mSUCCESS\e[0m"
+    else
+        FAILED+=("$NAME")
+        STATUS=$((STATUS + 1))
+        echo -e "DONE: \e[31mFAIL\e[0m"
+    fi
+
+done < <(find /mnt/chip -type f -executable ! -name runner.sh)
+
+if [ ! "$STATUS" -eq 0 ]; then
+    echo
+    echo "### FAILED: ${FAILED[*]}"
+fi
+
+exit "$STATUS"
diff --git a/src/test_driver/tizen/integration_tests/lighting-app/BUILD.gn b/src/test_driver/tizen/integration_tests/lighting-app/BUILD.gn
@@ -17,7 +17,7 @@ import("//build_overrides/tizen.gni")
 
 import("${tizen_sdk_build_root}/tizen_sdk.gni")
 
-tizen_qemu_mkisofs("test-runner") {
+tizen_qemu_mkisofs("lighting-app-runner") {
   runner = "runner.sh"
 
   # Build applications used in the test.
@@ -34,13 +34,18 @@ tizen_qemu_mkisofs("test-runner") {
   ]
 }
 
-tizen_qemu_run("check") {
+tizen_qemu_run("lighting-app") {
   # Enable network support, so Tizen can obtain current date/time from the
   # network. Correct date/time is required for the commissioning process -
   # attestation will fail otherwise.
   virtio_net = true
 
-  deps = [ ":test-runner" ]
-  mkisofs_outputs = get_target_outputs(":test-runner")
+  deps = [ ":lighting-app-runner" ]
+  mkisofs_outputs = get_target_outputs(":lighting-app-runner")
   iso_image = rebase_path(mkisofs_outputs[0])
 }
+
+group("check") {
+  deps = [ ":lighting-app" ]
+  testonly = true
+}
diff --git a/third_party/tizen/tizen_sdk.gni b/third_party/tizen/tizen_sdk.gni
@@ -198,6 +198,9 @@ template("tizen_qemu_mkisofs") {
       args += invoker.assets
       inputs += invoker.assets
     }
+    if (defined(invoker.assets_non_tracked)) {
+      args += invoker.assets_non_tracked
+    }
 
     script = "${build_root}/gn_run_binary.py"
   }