feat: enable go/python binding on mac/win (#378)

* feat: enable go/python binding on mac/win

* feat: enable go/python binding on mac/win

* feat: enable go/python binding on mac/win

* feat: enable go/python binding on mac/win

* feat: enable go/python binding on mac/win

* feat: enable go/python binding on mac/win

* feat: enable go/python binding on mac/win

* feat: enable go/python binding on mac/win

* feat: enable go/python binding on mac/win

* feat: enable go/python binding on mac/win

* feat: enable go/python binding on mac/win

* feat: enable go/python binding on mac/win

* feat: enable go/python binding on mac/win

* feat: enable go/python binding on mac/win

.github/workflows/build_mac.yml

@@ -76,7 +76,7 @@ jobs:
           fi
           export PATH=$(pwd)/core/ten_gn:$PATH
           echo $PATH
-          tgn gen mac arm64 ${{ matrix.build_type }} -- log_level=1 enable_serialized_actions=true ten_enable_private_tests=false ten_rust_enable_gen_cargo_config=false ten_enable_cargo_clean=true
+          tgn gen mac arm64 ${{ matrix.build_type }} -- log_level=1 enable_serialized_actions=true ten_enable_private_tests=false ten_rust_enable_gen_cargo_config=false ten_enable_cargo_clean=true ten_enable_python_binding=false
           tgn build mac arm64 ${{ matrix.build_type }}
           tree -I 'gen|obj' out
 
@@ -238,7 +238,7 @@ jobs:
           fi
           export PATH=$(pwd)/core/ten_gn:$PATH
           echo $PATH
-          tgn gen mac x64 ${{ matrix.build_type }} -- log_level=1 enable_serialized_actions=true ten_enable_private_tests=false ten_rust_enable_gen_cargo_config=false ten_enable_cargo_clean=true
+          tgn gen mac x64 ${{ matrix.build_type }} -- log_level=1 enable_serialized_actions=true ten_enable_private_tests=false ten_rust_enable_gen_cargo_config=false ten_enable_cargo_clean=true ten_enable_python_binding=false
           tgn build mac x64 ${{ matrix.build_type }}
           tree -I 'gen|obj' out
 
@@ -397,7 +397,7 @@ jobs:
           fi
           export PATH=$(pwd)/core/ten_gn:$PATH
           echo $PATH
-          tgn gen mac arm64 ${{ matrix.build_type }} -- log_level=1 enable_serialized_actions=true ten_enable_private_tests=false ten_rust_enable_gen_cargo_config=false ten_enable_ten_rust_apis=false
+          tgn gen mac arm64 ${{ matrix.build_type }} -- log_level=1 enable_serialized_actions=true ten_enable_private_tests=false ten_rust_enable_gen_cargo_config=false ten_enable_ten_rust_apis=false ten_enable_python_binding=false
           tgn build mac arm64 ${{ matrix.build_type }}
           tree -I 'gen|obj' out
 

.github/workflows/build_win.yml

@@ -46,10 +46,26 @@ jobs:
           rustup default nightly
           cargo install --force cbindgen
 
+      - name: Get Python executable path
+        run: |
+          $pythonPath = python -c "import sys; print(sys.executable)"
+          Write-Output "Python executable path: $pythonPath"
+
+          $pythonDir = Split-Path $pythonPath
+          Write-Output "Python directory path: $pythonDir"
+
+          echo "PYTHON3_PATH=$pythonDir" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+        shell: pwsh
+
+      - name: Use Python path
+        run: |
+          Write-Output "The Python directory is located at: $env:PYTHON3_PATH"
+        shell: pwsh
+
       - name: Build
         run: |
           $ENV:PATH += ";$PWD/core/ten_gn"
-          tgn gen win x64 ${{ matrix.build_type }} -- vs_version=2022 log_level=1 enable_serialized_actions=true ten_enable_private_tests=false ten_rust_enable_gen_cargo_config=false ten_enable_cargo_clean=true
+          tgn gen win x64 ${{ matrix.build_type }} -- vs_version=2022 log_level=1 enable_serialized_actions=true ten_enable_private_tests=false ten_rust_enable_gen_cargo_config=false ten_enable_cargo_clean=true ten_enable_python_binding=false ten_enable_go_binding=false
           tgn build win x64 ${{ matrix.build_type }}
 
       - name: Run Tests (ten_utils_unit_test)

build/common/python/python_config.py

@@ -5,8 +5,9 @@
 # Refer to the "LICENSE" file in the root directory for more information.
 #
 import argparse
-import os
 import sys
+import sysconfig
+import platform
 from build.scripts import cmd_exec
 
 
@@ -19,6 +20,91 @@ def __init__(self):
         self.log_level: int
 
 
+def get_embed_flags():
+    config = sysconfig.get_config_vars()
+
+    # Include directories.
+    include_dirs = config.get("INCLUDEPY", "")
+
+    # Library directories.
+    lib_dirs = config.get("LIBDIR", "")
+    # On some builds, you might need to add 'LIBPL' or similar.
+    lib_dirs_extra = config.get("LIBPL", "")
+    if lib_dirs_extra:
+        lib_dirs += f" {lib_dirs_extra}"
+
+    # Libraries.
+    libs = config.get("LIBS", "")
+    # Include the Python library.
+    python_lib = config.get("LIBRARY", "").replace("lib", "").replace(".a", "")
+    if python_lib:
+        libs += f" {python_lib}"
+
+    # C Flags.
+    cflags = config.get("CFLAGS", "")
+
+    return {
+        "include_dirs": include_dirs,
+        "lib_dirs": lib_dirs,
+        "libs": libs.strip(),
+        "cflags": cflags.strip(),
+    }
+
+
+def transform_flags_for_windows(embed_flags):
+    transformed = {"include_dirs": [], "lib_dirs": [], "libs": [], "cflags": []}
+
+    # Transform include directories.
+    for include_dir in embed_flags["include_dirs"].split():
+        transformed["include_dirs"].append(f'/I"{include_dir}"')
+
+    # Transform library directories.
+    for lib_dir in embed_flags["lib_dirs"].split():
+        transformed["lib_dirs"].append(f'/LIBPATH:"{lib_dir}"')
+
+    # Transform libraries.
+    for lib in embed_flags["libs"].split():
+        # Remove '-l' if present.
+        if lib.startswith("-l"):
+            lib = lib[2:]
+        # Append .lib extension if not present.
+        if not lib.endswith(".lib"):
+            lib = f'"{lib}.lib"'
+        else:
+            lib = f'"{lib}"'
+        transformed["libs"].append(lib)
+
+    # Transform CFLAGS.
+    transformed["cflags"] = embed_flags["cflags"].split()
+
+    return transformed
+
+
+def get_config_flags(config_type: str):
+    config = sysconfig.get_config_vars()
+
+    if config_type == "cflags":
+        return config.get("CFLAGS", "")
+    elif config_type == "ldflags":
+        return config.get("LDFLAGS", "")
+    elif config_type == "libs":
+        libs = config.get("LIBS", "")
+        # Remove '-l' prefixes if necessary.
+        libs = libs.replace("-l", "")
+        return libs
+    else:
+        raise ValueError(f"Unknown config_type: {config_type}")
+
+
+def build_cmd(config_type):
+    if config_type in ["cflags", "ldflags", "libs"]:
+        flag = get_config_flags(config_type)
+        cmd = flag.split()
+        return cmd
+    else:
+        raise ValueError(f"Unknown config_type: {config_type}")
+
+
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
 
@@ -30,50 +116,140 @@ def __init__(self):
     arg_info = ArgumentInfo()
     args = parser.parse_args(namespace=arg_info)
 
+    returncode = 0
+
     try:
         if args.target_os == "win":
-            py_path = os.environ.get("PYTHON3_PATH")
-            if py_path is None:
-                raise SystemError("PYTHON3_PATH is not provided")
+            # Retrieve embed flags using sysconfig.
+            embed_flags = get_embed_flags()
+            transformed_flags = transform_flags_for_windows(embed_flags)
+
+            if args.config_type == "cflags":
+                # Print include directories and any additional CFLAGS.
+                for flag in transformed_flags["include_dirs"]:
+                    print(flag)
+                for flag in transformed_flags["cflags"]:
+                    print(flag)
+            elif args.config_type == "ldflags":
+                # Print library directories.
+                for lib_dir in transformed_flags["lib_dirs"]:
+                    print(lib_dir)
+            elif args.config_type == "libs":
+                # Print libraries.
+                for lib in transformed_flags["libs"]:
+                    print(lib)
             else:
-                if args.config_type == "cflags":
-                    print('/I"' + os.path.join(py_path, "include") + '"')
-                elif args.config_type == "ldflags":
-                    print('/LIBPATH:"' + os.path.join(py_path, "libs" + '"'))
-                elif args.config_type == "libs":
-                    print('/Libs:"' + os.path.join(py_path, "libs" + '"'))
-                else:
-                    raise Exception(f"Unknown option: {args.config_type}")
+                raise Exception(f"Unknown config_type: {args.config_type}")
         else:
+            is_mac = platform.system().lower() == "darwin"
+
             if args.config_type == "cflags":
+                # For mac, same as Linux: just run python-config --embed
+                # --cflags
                 cmd = [
                     f"python{args.python_version}-config",
                     "--embed",
                     "--cflags",
                 ]
+
+                returncode, output_text = cmd_exec.run_cmd(cmd, args.log_level)
+                if returncode:
+                    raise Exception("Failed to run python-config.")
+
+                outputs = output_text.split()
+                for output in outputs:
+                    print(output)
+
             elif args.config_type == "ldflags":
-                cmd = [
+                # On mac, in addition to python-config --embed --ldflags, we
+                # also need to add the framework pairs extracted from libs.
+                cmd_ld = [
                     f"python{args.python_version}-config",
                     "--embed",
                     "--ldflags",
                 ]
+
+                returncode, ld_output = cmd_exec.run_cmd(cmd_ld, args.log_level)
+                if returncode:
+                    raise Exception("Failed to run python-config.")
+
+                ld_outputs = ld_output.split()
+
+                if is_mac:
+                    # Extract framework pairs from libs.
+                    cmd_libs = [
+                        f"python{args.python_version}-config",
+                        "--embed",
+                        "--libs",
+                    ]
+                    returncode, libs_output = cmd_exec.run_cmd(
+                        cmd_libs, args.log_level
+                    )
+                    if returncode:
+                        raise Exception("Failed to run python-config for libs.")
+
+                    libs_outputs = libs_output.split()
+
+                    frameworks_pairs = []
+                    skip_next = False
+                    prev_token = None
+                    for token in libs_outputs:
+                        if skip_next:
+                            # 'prev_token' was '-framework', 'token' is the
+                            # framework name.
+                            frameworks_pairs.append((prev_token, token))
+                            skip_next = False
+                            continue
+                        if token == "-framework":
+                            prev_token = token
+                            skip_next = True
+
+                    # Add the framework pairs to ldflags.
+                    for pair in frameworks_pairs:
+                        ld_outputs.append(pair[0])
+                        ld_outputs.append(pair[1])
+
+                for out in ld_outputs:
+                    print(out)
+
             elif args.config_type == "libs":
+                # On mac, we need to remove -framework <something> pairs from
+                # the output. On linux, just remove the '-l' prefix.
                 cmd = [
                     f"python{args.python_version}-config",
                     "--embed",
                     "--libs",
-                    "| sed 's/-l//g'",  # remove -l prefix
                 ]
-            else:
-                raise Exception(f"Unknown option: {args.config_type}")
 
-            returncode, output_text = cmd_exec.run_cmd(cmd, args.log_level)
-            if returncode:
-                raise Exception("Failed to run python-config.")
+                returncode, output_text = cmd_exec.run_cmd(cmd, args.log_level)
+                if returncode:
+                    raise Exception("Failed to run python-config.")
 
-            outputs = output_text.split()
-            for output in outputs:
-                print(output)
+                outputs = output_text.split()
+
+                if is_mac:
+                    filtered_libs = []
+                    skip_next = False
+                    for token in outputs:
+                        if skip_next:
+                            skip_next = False
+                            continue
+                        if token == "-framework":
+                            skip_next = True
+                        else:
+                            filtered_libs.append(
+                                token.replace("-l", "")
+                            )  # remove -l
+                    for lib in filtered_libs:
+                        print(lib)
+                else:
+                    # Linux-like behavior.
+                    filtered_libs = [lib.replace("-l", "") for lib in outputs]
+                    for lib in filtered_libs:
+                        print(lib)
+
+            else:
+                raise Exception(f"Unknown option: {args.config_type}")
 
     except Exception as exc:
         print(exc)

build/ten_runtime/options.gni

@@ -20,12 +20,8 @@ declare_args() {
 
 # ten_runtime binding
 declare_args() {
-  # If set, also build go binding
-  ten_enable_go_binding = is_mac || is_linux
-
-  # TODO: enable it on mac and win
-  ten_enable_python_binding =
-      is_linux && (target_cpu == "x64" || target_cpu == "arm64")
+  ten_enable_go_binding = true
+  ten_enable_python_binding = true
 }
 
 # ten_runtime extensions

core/include_internal/ten_runtime/binding/go/internal/common.h

@@ -30,14 +30,14 @@ TEN_RUNTIME_PRIVATE_API void ten_go_bridge_destroy_go_part(
     ten_go_bridge_t *self);
 
 TEN_RUNTIME_PRIVATE_API void ten_go_error_init_with_errno(ten_go_error_t *self,
-                                                          ten_errno_t errno);
+                                                          ten_errno_t err_no);
 
 TEN_RUNTIME_PRIVATE_API void ten_go_error_from_error(ten_go_error_t *self,
                                                      ten_error_t *err);
 
 TEN_RUNTIME_PRIVATE_API void ten_go_error_set_errno(ten_go_error_t *self,
-                                                    ten_errno_t errno);
+                                                    ten_errno_t err_no);
 
 TEN_RUNTIME_PRIVATE_API void ten_go_error_set(ten_go_error_t *self,
-                                              ten_errno_t errno,
+                                              ten_errno_t err_no,
                                               const char *msg);

core/src/ten_runtime/binding/go/interface/ten/common.h

@@ -40,7 +40,7 @@ typedef struct ten_go_bridge_t {
 typedef struct ten_go_error_t {
   // The errno is always 0 if no error.
   // The type of this field must equal to the errno of ten_error_t.
-  int64_t errno;
+  int64_t err_no;
 
   // The actually size of err_msg, not including the null terminator. It can be
   // used directly to determine if err_msg is empty in GO, without any cgo call.

core/src/ten_runtime/binding/go/interface/ten/error.go

@@ -36,15 +36,15 @@ func newTenError(errno uint32, errMsg string) error {
 // the `err_msg` in `status` will be freed after this function, do not access it
 // again.
 func withCGoError(status *C.ten_go_error_t) error {
-	if status.errno == 0 {
+	if status.err_no == 0 {
 		// No error.
 		return nil
 	}
 
 	if status.err_msg_size == 0 {
 		// An error occurred, but no error message.
 		return &TenError{
-			errno: uint32(status.errno),
+			errno: uint32(status.err_no),
 		}
 	}
 
@@ -54,7 +54,7 @@ func withCGoError(status *C.ten_go_error_t) error {
 	defer C.free(unsafe.Pointer(status.err_msg))
 
 	return &TenError{
-		errno:  uint32(status.errno),
+		errno:  uint32(status.err_no),
 		errMsg: C.GoString(status.err_msg),
 	}
 }