feat: clean up the pytest ci flow so that GitHub can launch pytest case

.github/workflows/build_linux_ubuntu2204.yml

@@ -38,14 +38,27 @@ jobs:
           python3 tools/version/update_version_in_ten_framework.py
           python3 tools/version/check_version_in_ten_framework.py
 
+      - name: Set ulimit and sysctl
+        run: |
+          # Because there are concurrent test cases which involves many
+          # concurrent socket connections, we need to enlarge the maximum number
+          # of the opened file descriptor.
+          ulimit -n 102400
+
+          # Because there are concurrent test cases (in 'smoke' and
+          # 'integration') which will create many concurrent connections
+          # simutaneously, we increase the TCP listening backlog value to 8192.
+          sysctl -w net.core.somaxconn=8192
+        shell: bash
+
       - name: Build
         run: |
           export PATH=$(pwd)/core/ten_gn:$PATH
           echo $PATH
           go env -w GOFLAGS="-buildvcs=false"
           go1.20.12 download
           rustup default nightly
-          tgn gen linux x64 ${{ matrix.build_type }} -- is_clang=${{ matrix.compiler == 'gcc' && 'false' || 'true' }} log_level=1 enable_serialized_actions=true ten_enable_integration_tests=false ten_enable_private_tests=false ten_rust_enable_gen_cargo_config=false ten_enable_cargo_clean=true
+          tgn gen linux x64 ${{ matrix.build_type }} -- is_clang=${{ matrix.compiler == 'gcc' && 'false' || 'true' }} log_level=1 enable_serialized_actions=true ten_enable_private_tests=false ten_rust_enable_gen_cargo_config=false ten_enable_cargo_clean=true
           tgn build linux x64 ${{ matrix.build_type }}
           tree -I 'gen|obj' out
 
@@ -126,6 +139,17 @@ jobs:
           cd out/linux/x64/
           pytest -s tests/ten_manager/
 
+      - name: Run tests (ten_runtime integration tests)
+        env:
+          ASAN_OPTIONS: detect_leaks=1:detect_stack_use_after_return=1:color=always:unmap_shadow_on_exit=1:abort_on_error=1
+          MALLOC_CHECK_: 3
+          TEN_ENABLE_MEMORY_TRACKING: "true"
+          TEN_ENABLE_BACKTRACE_DUMP: "true"
+        run: |
+          export PATH=$(pwd)/core/ten_gn:$PATH
+          cd out/linux/x64/
+          pytest -s tests/ten_runtime/integration/ || { echo "ten_runtime integration test failed"; exit 1; }
+
       - name: Update supports before upload or publish
         run: |
           UPDATE_SUPPORTS_SCRIPT=$(pwd)/tools/supports/update_supports_in_manifest_json.py

.github/workflows/build_mac.yml

@@ -76,7 +76,7 @@ jobs:
           fi
           export PATH=$(pwd)/core/ten_gn:$PATH
           echo $PATH
-          tgn gen mac arm64 ${{ matrix.build_type }} -- log_level=1 enable_serialized_actions=true ten_enable_integration_tests=false ten_enable_private_tests=false ten_rust_enable_gen_cargo_config=false ten_enable_cargo_clean=true
+          tgn gen mac arm64 ${{ matrix.build_type }} -- log_level=1 enable_serialized_actions=true ten_enable_private_tests=false ten_rust_enable_gen_cargo_config=false ten_enable_cargo_clean=true
           tgn build mac arm64 ${{ matrix.build_type }}
           tree -I 'gen|obj' out
 
@@ -220,7 +220,7 @@ jobs:
           fi
           export PATH=$(pwd)/core/ten_gn:$PATH
           echo $PATH
-          tgn gen mac x64 ${{ matrix.build_type }} -- log_level=1 enable_serialized_actions=true ten_enable_integration_tests=false ten_enable_private_tests=false ten_rust_enable_gen_cargo_config=false ten_enable_cargo_clean=true
+          tgn gen mac x64 ${{ matrix.build_type }} -- log_level=1 enable_serialized_actions=true ten_enable_private_tests=false ten_rust_enable_gen_cargo_config=false ten_enable_cargo_clean=true
           tgn build mac x64 ${{ matrix.build_type }}
           tree -I 'gen|obj' out
 
@@ -301,6 +301,17 @@ jobs:
           cd out/mac/x64/
           pytest -s tests/ten_manager/
 
+      - name: Run tests (ten_runtime integration tests)
+        env:
+          ASAN_OPTIONS: detect_stack_use_after_return=1:color=always:unmap_shadow_on_exit=1:abort_on_error=1
+          MALLOC_CHECK_: 3
+          TEN_ENABLE_MEMORY_TRACKING: "true"
+          TEN_ENABLE_BACKTRACE_DUMP: "true"
+        run: |
+          export PATH=$(pwd)/core/ten_gn:$PATH
+          cd out/mac/x64/
+          pytest -s tests/ten_runtime/integration/ || { echo "ten_runtime integration test failed"; exit 1; }
+
   # Building ten_runtime without the ten_rust library (i.e., ten_enable_ten_rust_apis=false).
   build-mac-arm64-without-ten-rust:
     concurrency:
@@ -365,7 +376,7 @@ jobs:
           fi
           export PATH=$(pwd)/core/ten_gn:$PATH
           echo $PATH
-          tgn gen mac arm64 ${{ matrix.build_type }} -- log_level=1 enable_serialized_actions=true ten_enable_integration_tests=false ten_enable_private_tests=false ten_rust_enable_gen_cargo_config=false ten_enable_ten_rust_apis=false
+          tgn gen mac arm64 ${{ matrix.build_type }} -- log_level=1 enable_serialized_actions=true ten_enable_private_tests=false ten_rust_enable_gen_cargo_config=false ten_enable_ten_rust_apis=false
           tgn build mac arm64 ${{ matrix.build_type }}
           tree -I 'gen|obj' out
 

.github/workflows/build_win.yml

@@ -49,7 +49,7 @@ jobs:
       - name: Build
         run: |
           $ENV:PATH += ";$PWD/core/ten_gn"
-          tgn gen win x64 ${{ matrix.build_type }} -- vs_version=2022 log_level=1 enable_serialized_actions=true ten_enable_integration_tests=false ten_enable_private_tests=false ten_rust_enable_gen_cargo_config=false ten_enable_cargo_clean=true
+          tgn gen win x64 ${{ matrix.build_type }} -- vs_version=2022 log_level=1 enable_serialized_actions=true ten_enable_private_tests=false ten_rust_enable_gen_cargo_config=false ten_enable_cargo_clean=true
           tgn build win x64 ${{ matrix.build_type }}
 
       - name: Run Tests (ten_utils_unit_test)

.vscode/settings.json

@@ -55,7 +55,8 @@
   ],
   "python.analysis.completeFunctionParens": true,
   "python.analysis.extraPaths": [
-    "./core/src/ten_runtime/binding/python/interface"
+    "./core/src/ten_runtime/binding/python/interface",
+    "./tests/ten_runtime/integration"
   ],
   "python.analysis.typeCheckingMode": "basic",
   "rust-analyzer.check.command": "clippy",

build/options.gni

@@ -18,4 +18,6 @@ declare_args() {
   ten_enable_package_manager = true
 
   ten_enable_cargo_clean = false
+
+  ten_enable_integration_tests_prebuilt = false
 }

build/ten_runtime/feature/build_pkg.py

@@ -178,30 +178,7 @@ def build_extension(args: ArgumentInfo) -> int:
     return returncode
 
 
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-
-    parser.add_argument("--pkg-src-root-dir", type=str, required=True)
-    parser.add_argument("--pkg-run-root-dir", type=str, required=False)
-    parser.add_argument("--tg-timestamp-proxy-file", type=str, required=True)
-    parser.add_argument("--pkg-name", type=str, required=True)
-    parser.add_argument("--pkg-language", type=str, required=True)
-    parser.add_argument("--os", type=str, required=True)
-    parser.add_argument("--cpu", type=str, required=True)
-    parser.add_argument("--build", type=str, required=True)
-    parser.add_argument("--tgn-path", type=str, required=False)
-    parser.add_argument("--is-clang", action=argparse.BooleanOptionalAction)
-    parser.add_argument(
-        "--enable-sanitizer", action=argparse.BooleanOptionalAction
-    )
-    parser.add_argument("--vs-version", type=str, required=False)
-    parser.add_argument(
-        "--log-level", type=int, required=True, help="specify log level"
-    )
-
-    arg_info = ArgumentInfo()
-    args = parser.parse_args(namespace=arg_info)
-
+def build(args: ArgumentInfo) -> int:
     if args.log_level > 0:
         msg = (
             f"> Start to build package({args.pkg_name})"
@@ -227,18 +204,51 @@ def build_extension(args: ArgumentInfo) -> int:
 
         # Success to build the app, update the stamp file to represent this
         # fact.
-        timestamp_proxy.touch_timestamp_proxy_file(args.tg_timestamp_proxy_file)
+        if args.tg_timestamp_proxy_file:
+            timestamp_proxy.touch_timestamp_proxy_file(
+                args.tg_timestamp_proxy_file
+            )
 
         if args.log_level > 0:
             print(f"Build {pkg_type}({args.pkg_name}) success")
 
     except Exception as e:
         returncode = 1
-        timestamp_proxy.remove_timestamp_proxy_file(
-            args.tg_timestamp_proxy_file
-        )
+        if args.tg_timestamp_proxy_file:
+            timestamp_proxy.remove_timestamp_proxy_file(
+                args.tg_timestamp_proxy_file
+            )
         print(f"Build package({args.pkg_name}) failed: {repr(e)}")
 
     finally:
         os.chdir(origin_wd)
-        sys.exit(returncode)
+
+    return returncode
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument("--pkg-src-root-dir", type=str, required=True)
+    parser.add_argument("--pkg-run-root-dir", type=str, required=False)
+    parser.add_argument("--tg-timestamp-proxy-file", type=str, required=False)
+    parser.add_argument("--pkg-name", type=str, required=True)
+    parser.add_argument("--pkg-language", type=str, required=True)
+    parser.add_argument("--os", type=str, required=True)
+    parser.add_argument("--cpu", type=str, required=True)
+    parser.add_argument("--build", type=str, required=True)
+    parser.add_argument("--tgn-path", type=str, required=False)
+    parser.add_argument("--is-clang", action=argparse.BooleanOptionalAction)
+    parser.add_argument(
+        "--enable-sanitizer", action=argparse.BooleanOptionalAction
+    )
+    parser.add_argument("--vs-version", type=str, required=False)
+    parser.add_argument(
+        "--log-level", type=int, required=True, help="specify log level"
+    )
+
+    arg_info = ArgumentInfo()
+    args = parser.parse_args(namespace=arg_info)
+
+    return_code = build(arg_info)
+    sys.exit(return_code)

build/ten_runtime/feature/prepare_integration_test_resources.py

@@ -0,0 +1,145 @@
+#
+# Copyright © 2024 Agora
+# This file is part of TEN Framework, an open source project.
+# Licensed under the Apache License, Version 2.0, with certain conditions.
+# Refer to the "LICENSE" file in the root directory for more information.
+#
+import argparse
+import json
+import sys
+import os
+from build.scripts import fs_utils
+
+assemble_info_dir = ".assemble_info"
+
+
+class ArgumentInfo(argparse.Namespace):
+    def __init__(self):
+        self.test_case_out_dir: str
+        self.test_case_src_dir: str
+        self.src_app: str
+        self.src_app_language: str
+        self.generated_app_src_root_dir_name: str
+        self.replace_files_after_install_app: list[str]
+        self.replace_files_after_install_all: list[str]
+
+
+def dump_integration_test_preparation_info_json(args: ArgumentInfo):
+    info = {
+        "test_case_src_dir": args.test_case_src_dir,
+        "src_app": args.src_app,
+        "src_app_language": args.src_app_language,
+        "generated_app_src_root_dir_name": args.generated_app_src_root_dir_name,
+        "replace_files_after_install_app": args.replace_files_after_install_app,
+        "replace_files_after_install_all": args.replace_files_after_install_all,
+    }
+
+    resource_dir = os.path.join(
+        args.test_case_out_dir,
+        assemble_info_dir,
+        args.generated_app_src_root_dir_name,
+    )
+    info_file = os.path.join(resource_dir, "info.json")
+    if not os.path.exists(resource_dir):
+        os.makedirs(resource_dir, exist_ok=True)
+
+    with open(info_file, "w", encoding="utf-8") as f:
+        json.dump(info, f, ensure_ascii=False, indent=2)
+
+
+def copy_replacement_files(
+    args: ArgumentInfo, replaced_files: list[str], dest_dir: str
+):
+    if replaced_files is None:
+        return
+
+    out_dir = os.path.join(
+        args.test_case_out_dir,
+        assemble_info_dir,
+        args.generated_app_src_root_dir_name,
+        dest_dir,
+    )
+
+    for file in replaced_files:
+        src_file = os.path.join(args.test_case_src_dir, file)
+        dst_file = os.path.join(out_dir, file)
+        if not os.path.exists(src_file):
+            print(f"File {src_file} does not exist")
+            continue
+
+        if not os.path.exists(os.path.dirname(dst_file)):
+            os.makedirs(os.path.dirname(dst_file), exist_ok=True)
+
+        fs_utils.copy(src_file, dst_file)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument(
+        "--test-case-out-dir",
+        type=str,
+        required=True,
+        help="Path to the test case output directory",
+    )
+
+    parser.add_argument(
+        "--test-case-src-dir",
+        type=str,
+        required=True,
+        help="Path to the test case source directory",
+    )
+
+    parser.add_argument(
+        "--src-app",
+        type=str,
+        required=True,
+        help="Path to the app source code",
+    )
+
+    parser.add_argument(
+        "--src-app-language",
+        type=str,
+        required=True,
+        help="Language of the app source code",
+    )
+
+    parser.add_argument(
+        "--generated-app-src-root-dir-name",
+        type=str,
+        required=True,
+        help="Name of the generated app source root directory",
+    )
+
+    parser.add_argument(
+        "--replace-files-after-install-app",
+        type=str,
+        action="append",
+        help="List of files to replace after installing app",
+    )
+
+    parser.add_argument(
+        "--replace-files-after-install-all",
+        type=str,
+        action="append",
+        help="List of files to replace after installing all",
+    )
+
+    arg_info = ArgumentInfo()
+    args = parser.parse_args(namespace=arg_info)
+
+    dump_integration_test_preparation_info_json(args)
+
+    copy_replacement_files(
+        args,
+        args.replace_files_after_install_app,
+        "files_to_be_replaced_after_install_app",
+    )
+
+    copy_replacement_files(
+        args,
+        args.replace_files_after_install_all,
+        "files_to_be_replaced_after_install_all",
+    )
+
+    sys.exit(0)