From 854703e3ad8cca1b7a7bd7e842121969f985b443 Mon Sep 17 00:00:00 2001 From: Hu Yueh-Wei Date: Mon, 9 Dec 2024 09:23:55 +0800 Subject: [PATCH] feat: enable go/python binding on mac/win (#378) * feat: enable go/python binding on mac/win * feat: enable go/python binding on mac/win * feat: enable go/python binding on mac/win * feat: enable go/python binding on mac/win * feat: enable go/python binding on mac/win * feat: enable go/python binding on mac/win * feat: enable go/python binding on mac/win * feat: enable go/python binding on mac/win * feat: enable go/python binding on mac/win * feat: enable go/python binding on mac/win * feat: enable go/python binding on mac/win * feat: enable go/python binding on mac/win * feat: enable go/python binding on mac/win * feat: enable go/python binding on mac/win --- .github/workflows/build_mac.yml | 6 +- .github/workflows/build_win.yml | 18 +- build/common/python/python_config.py | 220 ++++++++++++++++-- build/ten_runtime/options.gni | 8 +- .../ten_runtime/binding/go/internal/common.h | 6 +- .../binding/go/interface/ten/common.h | 2 +- .../binding/go/interface/ten/error.go | 6 +- .../binding/go/native/internal/common.c | 12 +- .../go/native/ten_env/ten_env_send_data.c | 1 - core/src/ten_runtime/output_libs.gni | 9 + core/src/ten_utils/sanitizer/memory_check.c | 9 +- core/ten_gn | 2 +- .../ffmpeg_demuxer/src/demuxer.cc | 2 +- .../ffmpeg_muxer/src/muxer.cc | 2 +- tests/ten_utils/common/test_utils.h | 4 +- 15 files changed, 252 insertions(+), 55 deletions(-) diff --git a/.github/workflows/build_mac.yml b/.github/workflows/build_mac.yml index ba7844cad..edbad6b05 100644 --- a/.github/workflows/build_mac.yml +++ b/.github/workflows/build_mac.yml @@ -76,7 +76,7 @@ jobs: fi export PATH=$(pwd)/core/ten_gn:$PATH echo $PATH - tgn gen mac arm64 ${{ matrix.build_type }} -- log_level=1 enable_serialized_actions=true ten_enable_private_tests=false ten_rust_enable_gen_cargo_config=false ten_enable_cargo_clean=true + tgn gen mac arm64 ${{ matrix.build_type }} -- log_level=1 enable_serialized_actions=true ten_enable_private_tests=false ten_rust_enable_gen_cargo_config=false ten_enable_cargo_clean=true ten_enable_python_binding=false tgn build mac arm64 ${{ matrix.build_type }} tree -I 'gen|obj' out @@ -238,7 +238,7 @@ jobs: fi export PATH=$(pwd)/core/ten_gn:$PATH echo $PATH - tgn gen mac x64 ${{ matrix.build_type }} -- log_level=1 enable_serialized_actions=true ten_enable_private_tests=false ten_rust_enable_gen_cargo_config=false ten_enable_cargo_clean=true + tgn gen mac x64 ${{ matrix.build_type }} -- log_level=1 enable_serialized_actions=true ten_enable_private_tests=false ten_rust_enable_gen_cargo_config=false ten_enable_cargo_clean=true ten_enable_python_binding=false tgn build mac x64 ${{ matrix.build_type }} tree -I 'gen|obj' out @@ -397,7 +397,7 @@ jobs: fi export PATH=$(pwd)/core/ten_gn:$PATH echo $PATH - tgn gen mac arm64 ${{ matrix.build_type }} -- log_level=1 enable_serialized_actions=true ten_enable_private_tests=false ten_rust_enable_gen_cargo_config=false ten_enable_ten_rust_apis=false + tgn gen mac arm64 ${{ matrix.build_type }} -- log_level=1 enable_serialized_actions=true ten_enable_private_tests=false ten_rust_enable_gen_cargo_config=false ten_enable_ten_rust_apis=false ten_enable_python_binding=false tgn build mac arm64 ${{ matrix.build_type }} tree -I 'gen|obj' out diff --git a/.github/workflows/build_win.yml b/.github/workflows/build_win.yml index 0d02c04cf..096cfab54 100644 --- a/.github/workflows/build_win.yml +++ b/.github/workflows/build_win.yml @@ -46,10 +46,26 @@ jobs: rustup default nightly cargo install --force cbindgen + - name: Get Python executable path + run: | + $pythonPath = python -c "import sys; print(sys.executable)" + Write-Output "Python executable path: $pythonPath" + + $pythonDir = Split-Path $pythonPath + Write-Output "Python directory path: $pythonDir" + + echo "PYTHON3_PATH=$pythonDir" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + shell: pwsh + + - name: Use Python path + run: | + Write-Output "The Python directory is located at: $env:PYTHON3_PATH" + shell: pwsh + - name: Build run: | $ENV:PATH += ";$PWD/core/ten_gn" - tgn gen win x64 ${{ matrix.build_type }} -- vs_version=2022 log_level=1 enable_serialized_actions=true ten_enable_private_tests=false ten_rust_enable_gen_cargo_config=false ten_enable_cargo_clean=true + tgn gen win x64 ${{ matrix.build_type }} -- vs_version=2022 log_level=1 enable_serialized_actions=true ten_enable_private_tests=false ten_rust_enable_gen_cargo_config=false ten_enable_cargo_clean=true ten_enable_python_binding=false ten_enable_go_binding=false tgn build win x64 ${{ matrix.build_type }} - name: Run Tests (ten_utils_unit_test) diff --git a/build/common/python/python_config.py b/build/common/python/python_config.py index 6ef5657f2..ab448d039 100644 --- a/build/common/python/python_config.py +++ b/build/common/python/python_config.py @@ -5,8 +5,9 @@ # Refer to the "LICENSE" file in the root directory for more information. # import argparse -import os import sys +import sysconfig +import platform from build.scripts import cmd_exec @@ -19,6 +20,91 @@ def __init__(self): self.log_level: int +def get_embed_flags(): + config = sysconfig.get_config_vars() + + # Include directories. + include_dirs = config.get("INCLUDEPY", "") + + # Library directories. + lib_dirs = config.get("LIBDIR", "") + # On some builds, you might need to add 'LIBPL' or similar. + lib_dirs_extra = config.get("LIBPL", "") + if lib_dirs_extra: + lib_dirs += f" {lib_dirs_extra}" + + # Libraries. + libs = config.get("LIBS", "") + # Include the Python library. + python_lib = config.get("LIBRARY", "").replace("lib", "").replace(".a", "") + if python_lib: + libs += f" {python_lib}" + + # C Flags. + cflags = config.get("CFLAGS", "") + + return { + "include_dirs": include_dirs, + "lib_dirs": lib_dirs, + "libs": libs.strip(), + "cflags": cflags.strip(), + } + + +def transform_flags_for_windows(embed_flags): + transformed = {"include_dirs": [], "lib_dirs": [], "libs": [], "cflags": []} + + # Transform include directories. + for include_dir in embed_flags["include_dirs"].split(): + transformed["include_dirs"].append(f'/I"{include_dir}"') + + # Transform library directories. + for lib_dir in embed_flags["lib_dirs"].split(): + transformed["lib_dirs"].append(f'/LIBPATH:"{lib_dir}"') + + # Transform libraries. + for lib in embed_flags["libs"].split(): + # Remove '-l' if present. + if lib.startswith("-l"): + lib = lib[2:] + # Append .lib extension if not present. + if not lib.endswith(".lib"): + lib = f'"{lib}.lib"' + else: + lib = f'"{lib}"' + transformed["libs"].append(lib) + + # Transform CFLAGS. + transformed["cflags"] = embed_flags["cflags"].split() + + return transformed + + +def get_config_flags(config_type: str): + config = sysconfig.get_config_vars() + + if config_type == "cflags": + return config.get("CFLAGS", "") + elif config_type == "ldflags": + return config.get("LDFLAGS", "") + elif config_type == "libs": + libs = config.get("LIBS", "") + # Remove '-l' prefixes if necessary. + libs = libs.replace("-l", "") + return libs + else: + raise ValueError(f"Unknown config_type: {config_type}") + + +def build_cmd(config_type): + if config_type in ["cflags", "ldflags", "libs"]: + flag = get_config_flags(config_type) + cmd = flag.split() + return cmd + else: + raise ValueError(f"Unknown config_type: {config_type}") + + if __name__ == "__main__": parser = argparse.ArgumentParser() @@ -30,50 +116,140 @@ def __init__(self): arg_info = ArgumentInfo() args = parser.parse_args(namespace=arg_info) + returncode = 0 + try: if args.target_os == "win": - py_path = os.environ.get("PYTHON3_PATH") - if py_path is None: - raise SystemError("PYTHON3_PATH is not provided") + # Retrieve embed flags using sysconfig. + embed_flags = get_embed_flags() + transformed_flags = transform_flags_for_windows(embed_flags) + + if args.config_type == "cflags": + # Print include directories and any additional CFLAGS. + for flag in transformed_flags["include_dirs"]: + print(flag) + for flag in transformed_flags["cflags"]: + print(flag) + elif args.config_type == "ldflags": + # Print library directories. + for lib_dir in transformed_flags["lib_dirs"]: + print(lib_dir) + elif args.config_type == "libs": + # Print libraries. + for lib in transformed_flags["libs"]: + print(lib) else: - if args.config_type == "cflags": - print('/I"' + os.path.join(py_path, "include") + '"') - elif args.config_type == "ldflags": - print('/LIBPATH:"' + os.path.join(py_path, "libs" + '"')) - elif args.config_type == "libs": - print('/Libs:"' + os.path.join(py_path, "libs" + '"')) - else: - raise Exception(f"Unknown option: {args.config_type}") + raise Exception(f"Unknown config_type: {args.config_type}") else: + is_mac = platform.system().lower() == "darwin" + if args.config_type == "cflags": + # For mac, same as Linux: just run python-config --embed + # --cflags cmd = [ f"python{args.python_version}-config", "--embed", "--cflags", ] + + returncode, output_text = cmd_exec.run_cmd(cmd, args.log_level) + if returncode: + raise Exception("Failed to run python-config.") + + outputs = output_text.split() + for output in outputs: + print(output) + elif args.config_type == "ldflags": - cmd = [ + # On mac, in addition to python-config --embed --ldflags, we + # also need to add the framework pairs extracted from libs. + cmd_ld = [ f"python{args.python_version}-config", "--embed", "--ldflags", ] + + returncode, ld_output = cmd_exec.run_cmd(cmd_ld, args.log_level) + if returncode: + raise Exception("Failed to run python-config.") + + ld_outputs = ld_output.split() + + if is_mac: + # Extract framework pairs from libs. + cmd_libs = [ + f"python{args.python_version}-config", + "--embed", + "--libs", + ] + returncode, libs_output = cmd_exec.run_cmd( + cmd_libs, args.log_level + ) + if returncode: + raise Exception("Failed to run python-config for libs.") + + libs_outputs = libs_output.split() + + frameworks_pairs = [] + skip_next = False + prev_token = None + for token in libs_outputs: + if skip_next: + # 'prev_token' was '-framework', 'token' is the + # framework name. + frameworks_pairs.append((prev_token, token)) + skip_next = False + continue + if token == "-framework": + prev_token = token + skip_next = True + + # Add the framework pairs to ldflags. + for pair in frameworks_pairs: + ld_outputs.append(pair[0]) + ld_outputs.append(pair[1]) + + for out in ld_outputs: + print(out) + elif args.config_type == "libs": + # On mac, we need to remove -framework pairs from + # the output. On linux, just remove the '-l' prefix. cmd = [ f"python{args.python_version}-config", "--embed", "--libs", - "| sed 's/-l//g'", # remove -l prefix ] - else: - raise Exception(f"Unknown option: {args.config_type}") - returncode, output_text = cmd_exec.run_cmd(cmd, args.log_level) - if returncode: - raise Exception("Failed to run python-config.") + returncode, output_text = cmd_exec.run_cmd(cmd, args.log_level) + if returncode: + raise Exception("Failed to run python-config.") - outputs = output_text.split() - for output in outputs: - print(output) + outputs = output_text.split() + + if is_mac: + filtered_libs = [] + skip_next = False + for token in outputs: + if skip_next: + skip_next = False + continue + if token == "-framework": + skip_next = True + else: + filtered_libs.append( + token.replace("-l", "") + ) # remove -l + for lib in filtered_libs: + print(lib) + else: + # Linux-like behavior. + filtered_libs = [lib.replace("-l", "") for lib in outputs] + for lib in filtered_libs: + print(lib) + + else: + raise Exception(f"Unknown option: {args.config_type}") except Exception as exc: print(exc) diff --git a/build/ten_runtime/options.gni b/build/ten_runtime/options.gni index 0b76fbf81..9333b8504 100644 --- a/build/ten_runtime/options.gni +++ b/build/ten_runtime/options.gni @@ -20,12 +20,8 @@ declare_args() { # ten_runtime binding declare_args() { - # If set, also build go binding - ten_enable_go_binding = is_mac || is_linux - - # TODO: enable it on mac and win - ten_enable_python_binding = - is_linux && (target_cpu == "x64" || target_cpu == "arm64") + ten_enable_go_binding = true + ten_enable_python_binding = true } # ten_runtime extensions diff --git a/core/include_internal/ten_runtime/binding/go/internal/common.h b/core/include_internal/ten_runtime/binding/go/internal/common.h index 011ac1512..23b549492 100644 --- a/core/include_internal/ten_runtime/binding/go/internal/common.h +++ b/core/include_internal/ten_runtime/binding/go/internal/common.h @@ -30,14 +30,14 @@ TEN_RUNTIME_PRIVATE_API void ten_go_bridge_destroy_go_part( ten_go_bridge_t *self); TEN_RUNTIME_PRIVATE_API void ten_go_error_init_with_errno(ten_go_error_t *self, - ten_errno_t errno); + ten_errno_t err_no); TEN_RUNTIME_PRIVATE_API void ten_go_error_from_error(ten_go_error_t *self, ten_error_t *err); TEN_RUNTIME_PRIVATE_API void ten_go_error_set_errno(ten_go_error_t *self, - ten_errno_t errno); + ten_errno_t err_no); TEN_RUNTIME_PRIVATE_API void ten_go_error_set(ten_go_error_t *self, - ten_errno_t errno, + ten_errno_t err_no, const char *msg); diff --git a/core/src/ten_runtime/binding/go/interface/ten/common.h b/core/src/ten_runtime/binding/go/interface/ten/common.h index c79377a37..7517dc439 100644 --- a/core/src/ten_runtime/binding/go/interface/ten/common.h +++ b/core/src/ten_runtime/binding/go/interface/ten/common.h @@ -40,7 +40,7 @@ typedef struct ten_go_bridge_t { typedef struct ten_go_error_t { // The errno is always 0 if no error. // The type of this field must equal to the errno of ten_error_t. - int64_t errno; + int64_t err_no; // The actually size of err_msg, not including the null terminator. It can be // used directly to determine if err_msg is empty in GO, without any cgo call. diff --git a/core/src/ten_runtime/binding/go/interface/ten/error.go b/core/src/ten_runtime/binding/go/interface/ten/error.go index b8d26aaf2..4ff5e2be5 100644 --- a/core/src/ten_runtime/binding/go/interface/ten/error.go +++ b/core/src/ten_runtime/binding/go/interface/ten/error.go @@ -36,7 +36,7 @@ func newTenError(errno uint32, errMsg string) error { // the `err_msg` in `status` will be freed after this function, do not access it // again. func withCGoError(status *C.ten_go_error_t) error { - if status.errno == 0 { + if status.err_no == 0 { // No error. return nil } @@ -44,7 +44,7 @@ func withCGoError(status *C.ten_go_error_t) error { if status.err_msg_size == 0 { // An error occurred, but no error message. return &TenError{ - errno: uint32(status.errno), + errno: uint32(status.err_no), } } @@ -54,7 +54,7 @@ func withCGoError(status *C.ten_go_error_t) error { defer C.free(unsafe.Pointer(status.err_msg)) return &TenError{ - errno: uint32(status.errno), + errno: uint32(status.err_no), errMsg: C.GoString(status.err_msg), } } diff --git a/core/src/ten_runtime/binding/go/native/internal/common.c b/core/src/ten_runtime/binding/go/native/internal/common.c index 5cef5b2a6..f8c4843d2 100644 --- a/core/src/ten_runtime/binding/go/native/internal/common.c +++ b/core/src/ten_runtime/binding/go/native/internal/common.c @@ -61,10 +61,10 @@ void ten_go_bridge_destroy_go_part(ten_go_bridge_t *self) { } } -void ten_go_error_init_with_errno(ten_go_error_t *self, ten_errno_t errno) { +void ten_go_error_init_with_errno(ten_go_error_t *self, ten_errno_t err_no) { TEN_ASSERT(self, "Should not happen."); - self->errno = errno; + self->err_no = err_no; self->err_msg_size = 0; self->err_msg = NULL; } @@ -75,17 +75,17 @@ void ten_go_error_from_error(ten_go_error_t *self, ten_error_t *err) { ten_go_error_set(self, ten_error_errno(err), ten_error_errmsg(err)); } -void ten_go_error_set_errno(ten_go_error_t *self, ten_errno_t errno) { +void ten_go_error_set_errno(ten_go_error_t *self, ten_errno_t err_no) { TEN_ASSERT(self, "Should not happen."); - self->errno = errno; + self->err_no = err_no; } -void ten_go_error_set(ten_go_error_t *self, ten_errno_t errno, +void ten_go_error_set(ten_go_error_t *self, ten_errno_t err_no, const char *msg) { TEN_ASSERT(self, "Should not happen."); - self->errno = errno; + self->err_no = err_no; if (msg == NULL || strlen(msg) == 0) { return; } diff --git a/core/src/ten_runtime/binding/go/native/ten_env/ten_env_send_data.c b/core/src/ten_runtime/binding/go/native/ten_env/ten_env_send_data.c index 39555e56c..ebf29199a 100644 --- a/core/src/ten_runtime/binding/go/native/ten_env/ten_env_send_data.c +++ b/core/src/ten_runtime/binding/go/native/ten_env/ten_env_send_data.c @@ -5,7 +5,6 @@ // Refer to the "LICENSE" file in the root directory for more information. // #include -#include #include "include_internal/ten_runtime/binding/go/internal/common.h" #include "include_internal/ten_runtime/binding/go/msg/msg.h" diff --git a/core/src/ten_runtime/output_libs.gni b/core/src/ten_runtime/output_libs.gni index bd65fb540..d27aee364 100644 --- a/core/src/ten_runtime/output_libs.gni +++ b/core/src/ten_runtime/output_libs.gni @@ -28,6 +28,15 @@ if (is_linux) { ten_runtime_go_output_libs = [ "${root_out_dir}/libten_runtime_go.dylib" ] ten_runtime_python_output_libs = [ "${root_out_dir}/libten_runtime_python.dylib" ] +} else if (is_win) { + ten_runtime_go_output_libs = [ + "${root_out_dir}/ten_runtime_go.dll", + "${root_out_dir}/ten_runtime_go.dll.lib", + ] + ten_runtime_python_output_libs = [ + "${root_out_dir}/ten_runtime_python.dll", + "${root_out_dir}/ten_runtime_python.dll.lib", + ] } template("ten_runtime_copy_deps") { diff --git a/core/src/ten_utils/sanitizer/memory_check.c b/core/src/ten_utils/sanitizer/memory_check.c index b6e03ff68..26eddb37f 100644 --- a/core/src/ten_utils/sanitizer/memory_check.c +++ b/core/src/ten_utils/sanitizer/memory_check.c @@ -123,15 +123,16 @@ static ten_sanitizer_memory_record_t *ten_sanitizer_memory_record_create( self->func_name = (char *)malloc(strlen(func_name) + 1); TEN_ASSERT(self->func_name, "Failed to allocate memory."); - (void)sprintf(self->func_name, "%s", func_name); + (void)snprintf(self->func_name, strlen(func_name) + 1, "%s", func_name); self->file_name = (char *)malloc(strlen(file_name) + 1); TEN_ASSERT(self->file_name, "Failed to allocate memory."); TEN_ASSERT(strlen(file_name) >= TEN_FILE_PATH_RELATIVE_PREFIX_LENGTH, "Should not happen."); - (void)sprintf(self->file_name, "%.*s", - (int)(strlen(file_name) - TEN_FILE_PATH_RELATIVE_PREFIX_LENGTH), - file_name + TEN_FILE_PATH_RELATIVE_PREFIX_LENGTH); + (void)snprintf( + self->file_name, strlen(file_name) + 1, "%.*s", + (int)(strlen(file_name) - TEN_FILE_PATH_RELATIVE_PREFIX_LENGTH), + file_name + TEN_FILE_PATH_RELATIVE_PREFIX_LENGTH); self->lineno = lineno; diff --git a/core/ten_gn b/core/ten_gn index bc375eedc..ce206d027 160000 --- a/core/ten_gn +++ b/core/ten_gn @@ -1 +1 @@ -Subproject commit bc375eedca6085a8a450614f3b8d4fb79c8b64bb +Subproject commit ce206d0278b9eae9e7b961b218aedb268f84d8e5 diff --git a/packages/example_extensions/ffmpeg_demuxer/src/demuxer.cc b/packages/example_extensions/ffmpeg_demuxer/src/demuxer.cc index 75bd0cbc0..69294f624 100644 --- a/packages/example_extensions/ffmpeg_demuxer/src/demuxer.cc +++ b/packages/example_extensions/ffmpeg_demuxer/src/demuxer.cc @@ -459,7 +459,7 @@ void save_img_frame(ten::video_frame_t &pFrame, int index) { int height = pFrame.get_height(); // Open file - (void)sprintf(szFilename, "frame%d.ppm", index); + (void)snprintf(szFilename, 32, "frame%d.ppm", index); pFile = fopen(szFilename, "wb"); if (pFile == nullptr) { diff --git a/packages/example_extensions/ffmpeg_muxer/src/muxer.cc b/packages/example_extensions/ffmpeg_muxer/src/muxer.cc index 386410904..4877518e9 100644 --- a/packages/example_extensions/ffmpeg_muxer/src/muxer.cc +++ b/packages/example_extensions/ffmpeg_muxer/src/muxer.cc @@ -750,7 +750,7 @@ TEN_UNUSED static void save_img_frame( int height = pFrame->get_height(); // Open file - (void)sprintf(szFilename, "encode_frame%d.ppm", index); + (void)snprintf(szFilename, 32, "encode_frame%d.ppm", index); pFile = fopen(szFilename, "wb"); if (pFile == nullptr) { diff --git a/tests/ten_utils/common/test_utils.h b/tests/ten_utils/common/test_utils.h index 070fdeb46..8965f5df8 100644 --- a/tests/ten_utils/common/test_utils.h +++ b/tests/ten_utils/common/test_utils.h @@ -31,8 +31,8 @@ static std::string ToHex(const uint8_t *data, size_t size) { char buf[3] = {0}; std::string s; for (int i = 0; i < size; i++) { - sprintf(buf, "%.2x", data[i]); + snprintf(buf, 3, "%.2x", data[i]); s += buf; } return s; -} \ No newline at end of file +}