feat!: add addon registration func mechanism for python (#342)

core/include/ten_runtime/addon/extension/extension.h

@@ -20,9 +20,20 @@ typedef struct ten_addon_t ten_addon_t;
 typedef struct ten_extension_t ten_extension_t;
 typedef struct ten_addon_host_t ten_addon_host_t;
 
+typedef ten_addon_host_t *(*ten_addon_register_extension_func_t)(
+    const char *name, const char *base_dir, ten_addon_t *addon);
+
+typedef ten_addon_host_t *(*ten_addon_register_extension_v2_func_t)(
+    const char *name, const char *base_dir, ten_addon_t *addon,
+    void *register_ctx);
+
 TEN_RUNTIME_API ten_addon_host_t *ten_addon_register_extension(
     const char *name, const char *base_dir, ten_addon_t *addon);
 
+TEN_RUNTIME_API ten_addon_host_t *ten_addon_register_extension_v2(
+    const char *name, const char *base_dir, ten_addon_t *addon,
+    void *register_ctx);
+
 TEN_RUNTIME_API ten_addon_t *ten_addon_unregister_extension(const char *name);
 
 TEN_RUNTIME_API bool ten_addon_create_extension(

core/include_internal/ten_runtime/addon/extension/extension.h

@@ -12,8 +12,3 @@ typedef struct ten_addon_store_t ten_addon_store_t;
 typedef struct ten_addon_t ten_addon_t;
 
 TEN_RUNTIME_PRIVATE_API ten_addon_store_t *ten_extension_get_global_store(void);
-
-TEN_RUNTIME_API void ten_addon_register_extension_v2(const char *name,
-                                                     const char *base_dir,
-                                                     void *register_ctx,
-                                                     ten_addon_t *addon);

core/include_internal/ten_runtime/binding/python/addon/decorator.h

@@ -19,5 +19,13 @@ typedef struct ten_py_decorator_register_addon_t {
   ten_string_t base_dir;
 } ten_py_decorator_register_addon_t;
 
+typedef struct ten_py_decorator_register_addon_v2_t {
+  PyObject_HEAD
+} ten_py_decorator_register_addon_v2_t;
+
 TEN_RUNTIME_PRIVATE_API bool
 ten_py_decorator_register_addon_as_extension_init_for_module(PyObject *module);
+
+TEN_RUNTIME_PRIVATE_API bool
+ten_py_decorator_register_addon_as_extension_init_for_module_v2(
+    PyObject *module);

core/include_internal/ten_runtime/engine/internal/remote_interface.h

@@ -25,19 +25,6 @@ typedef struct ten_engine_on_protocol_created_info_t {
   void *user_data;
 } ten_engine_on_protocol_created_info_t;
 
-TEN_RUNTIME_PRIVATE_API ten_engine_on_protocol_created_info_t *
-ten_engine_on_protocol_created_info_create(ten_engine_on_remote_created_cb_t cb,
-                                           void *user_data);
-
-TEN_RUNTIME_PRIVATE_API void ten_engine_on_protocol_created_info_destroy(
-    ten_engine_on_protocol_created_info_t *self);
-
-TEN_RUNTIME_PRIVATE_API void ten_engine_add_remote(ten_engine_t *self,
-                                                   ten_remote_t *remote);
-
-TEN_RUNTIME_PRIVATE_API void ten_engine_add_weak_remote(ten_engine_t *self,
-                                                        ten_remote_t *remote);
-
 TEN_RUNTIME_PRIVATE_API void ten_engine_upgrade_weak_remote_to_normal_remote(
     ten_engine_t *self, ten_remote_t *remote);
 
@@ -62,8 +49,5 @@ TEN_RUNTIME_PRIVATE_API void ten_engine_on_remote_closed(ten_remote_t *remote,
 TEN_RUNTIME_PRIVATE_API bool ten_engine_receive_msg_from_remote(
     ten_remote_t *remote, ten_shared_ptr_t *msg, void *user_data);
 
-TEN_RUNTIME_PRIVATE_API ten_remote_t *ten_engine_find_remote(ten_engine_t *self,
-                                                             const char *uri);
-
 TEN_RUNTIME_PRIVATE_API void ten_engine_link_connection_to_remote(
     ten_engine_t *self, ten_connection_t *connection, const char *uri);

core/src/ten_runtime/addon/extension/extension.c

@@ -15,7 +15,6 @@
 #include "include_internal/ten_runtime/ten_env/ten_env.h"
 #include "ten_runtime/addon/addon.h"
 #include "ten_runtime/ten_env/ten_env.h"
-#include "ten_utils/backtrace/backtrace.h"
 #include "ten_utils/macro/check.h"
 #include "ten_utils/macro/mark.h"
 
@@ -139,19 +138,27 @@ ten_addon_host_t *ten_addon_register_extension(const char *name,
   return addon_host;
 }
 
-// TODO(Wei): Reconsider the `register` and `unregister` mechanisms of the
-// addon.
-void ten_addon_register_extension_v2(const char *name, const char *base_dir,
-                                     void *register_ctx, ten_addon_t *addon) {
+ten_addon_host_t *ten_addon_register_extension_v2(const char *name,
+                                                  const char *base_dir,
+                                                  ten_addon_t *addon,
+                                                  void *register_ctx) {
   if (!name || strlen(name) == 0) {
     TEN_LOGE("The addon name is required.");
     exit(EXIT_FAILURE);
   }
 
   ten_addon_host_t *addon_host = (ten_addon_host_t *)register_ctx;
 
+  // If no `addon_host` is provided, create one here. Whether it is created here
+  // or received, pass it out in the end.
+  if (!addon_host) {
+    addon_host = ten_addon_host_create(TEN_ADDON_TYPE_EXTENSION);
+  }
+
   ten_addon_register(ten_extension_get_global_store(), addon_host, name,
                      base_dir, addon);
+
+  return addon_host;
 }
 
 ten_addon_t *ten_addon_unregister_extension(const char *name) {

core/src/ten_runtime/binding/python/interface/ten/__init__.py

@@ -11,6 +11,7 @@
 from .addon import Addon
 from .decorator import (
     register_addon_as_extension,
+    register_addon_as_extension_v2,
 )
 from .ten_env import TenEnv
 from .cmd import Cmd
@@ -26,6 +27,7 @@
 __all__ = [
     "Addon",
     "register_addon_as_extension",
+    "register_addon_as_extension_v2",
     "App",
     "Extension",
     "AsyncExtension",

core/src/ten_runtime/binding/python/interface/ten/addon.py

@@ -8,13 +8,21 @@
 import os
 import importlib.util
 from glob import glob
+from typing import Callable, Any, Dict
 from libten_runtime_python import _Addon
 from .ten_env import TenEnv
 
 
-class Addon(_Addon):
+class _AddonManager:
+    # Use the simple approach below, similar to a global array, to detect
+    # whether a Python module provides the registration function required by the
+    # TEN runtime. This avoids using `setattr` on the module, which may not be
+    # supported in advanced environments like Cython. The global array method
+    # is simple enough that it should work in all environments.
+    _registration_registry: Dict[str, Callable[[Any], None]] = {}
+
     @classmethod
-    def _load_all(cls):
+    def _load_all(cls, register_ctx: object):
         base_dir = cls._find_app_base_dir()
 
         # Read manifest.json under base_dir.
@@ -47,28 +55,105 @@ def _load_all(cls):
                 module_name = os.path.basename(module)
 
                 if module_name in extension_names:
-                    # Proceed to load the module.
-                    spec = importlib.util.find_spec(
-                        "ten_packages.extension.{}".format(module_name)
+                    cls._load_module(
+                        module_full_name=(
+                            f"ten_packages.extension.{module_name}"
+                        ),
+                        module_name=module_name,
+                        register_ctx=register_ctx,
                     )
-                    if spec is not None:
-                        _ = importlib.import_module(
-                            "ten_packages.extension.{}".format(module_name)
-                        )
-                        print("imported module: {}".format(module_name))
                 else:
-                    print("Skipping module: {}".format(module_name))
+                    print(f"Skipping module: {module_name}")
 
     @classmethod
-    def _load_from_path(cls, path):
-        module_name = os.path.basename(path)
-        spec = importlib.util.find_spec(module_name)
-        if spec is not None:
-            mod = importlib.import_module(module_name)
+    def _load_module(
+        cls,
+        module_full_name: str,
+        module_name: str,
+        register_ctx: object,
+    ):
+        """
+        Helper method to load a module, check for the special function,
+        invoke it, and unload the module if the special function is missing.
+        """
+        try:
+            spec = importlib.util.find_spec(module_full_name)
+            if spec is None:
+                raise ImportError(f"Cannot find module: {module_full_name}")
+
+            _ = importlib.import_module(module_full_name)
             print(f"Imported module: {module_name}")
-            return mod
-        else:
-            raise ImportError(f"Cannot find module: {module_name}")
+
+            # Retrieve the registration function from the global registry
+            registration_func_name = _AddonManager._get_registration_func_name(
+                module_name
+            )
+
+            registration_func = _AddonManager._get_registration_func(
+                module_name
+            )
+
+            if registration_func:
+                try:
+                    registration_func(register_ctx)
+                    print(f"Successfully registered addon '{module_name}'")
+                except Exception as e:
+                    print(
+                        (
+                            "Error during registration of addon "
+                            f"'{module_name}': {e}"
+                        )
+                    )
+                finally:
+                    # Remove the registration function from the global registry.
+                    if (
+                        registration_func_name
+                        in _AddonManager._registration_registry
+                    ):
+                        del _AddonManager._registration_registry[
+                            registration_func_name
+                        ]
+                        print(
+                            (
+                                "Removed registration function for addon "
+                                f"'{registration_func_name}'"
+                            )
+                        )
+            else:
+                print(f"No {registration_func_name} found in {module_name}")
+
+        except ImportError as e:
+            print(f"Error importing module {module_name}: {e}")
+
+    @staticmethod
+    def _get_registration_func_name(addon_name: str) -> str:
+        return f"____ten_addon_{addon_name}_register____"
+
+    @staticmethod
+    def _get_registration_func(addon_name: str) -> Callable[[Any], None] | None:
+        return _AddonManager._registration_registry.get(
+            _AddonManager._get_registration_func_name(addon_name)
+        )
+
+    @staticmethod
+    def _set_registration_func(
+        addon_name: str,
+        registration_func: Callable[[Any], None],
+    ) -> None:
+        registration_func_name = _AddonManager._get_registration_func_name(
+            addon_name
+        )
+
+        print(
+            (
+                f"Injected registration function '{registration_func_name}' "
+                "into module '{module.__name__}'"
+            )
+        )
+
+        _AddonManager._registration_registry[registration_func_name] = (
+            registration_func
+        )
 
     @staticmethod
     def _find_app_base_dir():
@@ -92,12 +177,10 @@ def _find_app_base_dir():
             "App base directory with a valid manifest.json not found."
         )
 
+
+class Addon(_Addon):
     def on_init(self, ten_env: TenEnv) -> None:
         ten_env.on_init_done()
 
     def on_deinit(self, ten_env: TenEnv) -> None:
         ten_env.on_deinit_done()
-
-    def on_create_instance(
-        self, ten_env: TenEnv, name: str, context
-    ) -> None: ...

core/src/ten_runtime/binding/python/interface/ten/cmd.py

@@ -4,7 +4,6 @@
 # Licensed under the Apache License, Version 2.0, with certain conditions.
 # Refer to the "LICENSE" file in the root directory for more information.
 #
-import json
 from libten_runtime_python import _Cmd
 
 

core/src/ten_runtime/binding/python/interface/ten/decorator.py

@@ -6,8 +6,11 @@
 #
 import os
 import sys
+from typing import Type
+from .addon import _AddonManager, Addon
 from libten_runtime_python import (
     _register_addon_as_extension,
+    _register_addon_as_extension_v2,
 )
 
 
@@ -27,3 +30,66 @@ def register_addon_as_extension(name: str, base_dir: str | None = None):
         base_dir = os.path.dirname(base_dir)
 
     return _register_addon_as_extension(name, base_dir)
+
+
+def register_addon_as_extension_v2(name: str, base_dir: str | None = None):
+    """
+    Decorator to register a class as an addon extension and create a special
+    registration function required by the Addon loader.
+
+    Args:
+        name (str): The name of the addon extension.
+        base_dir (str, optional): The base directory of the addon. Defaults to
+            None.
+
+    Returns:
+        Callable: The decorator function.
+    """
+
+    def decorator(cls: Type[Addon]) -> Type[Addon]:
+        # Resolve base_dir.
+        if base_dir is None:
+            try:
+                # Attempt to get the caller's file path using sys._getframe()
+                caller_frame = sys._getframe(1)
+                resolved_base_dir = os.path.dirname(
+                    caller_frame.f_code.co_filename
+                )
+            except (AttributeError, ValueError):
+                # Fallback in case sys._getframe() is not available or fails.
+                # Example: in Cython or restricted environments.
+                resolved_base_dir = None
+        else:
+            # If base_dir is provided, ensure it's the directory name
+            resolved_base_dir = os.path.dirname(base_dir)
+
+        # Define the registration function that will be called by the Addon
+        # loader.
+        def registration_func(register_ctx):
+            """
+            Registration function injected into the module to handle addon
+            registration.
+
+            Args:
+                register_ctx: An opaque parameter provided by the Addon loader.
+            """
+            # Instantiate the addon class.
+            instance = cls()
+
+            try:
+                _register_addon_as_extension_v2(
+                    name, resolved_base_dir, instance, register_ctx
+                )
+                print(
+                    f"Called '_register_addon_as_extension' for addon '{name}'"
+                )
+            except Exception as e:
+                print(f"Failed to register addon '{name}': {e}")
+
+        # Define the registration function name based on the addon name.
+        _AddonManager._set_registration_func(name, registration_func)
+
+        # Return the original class without modification.
+        return cls
+
+    return decorator

core/src/ten_runtime/binding/python/interface/ten/libten_runtime_python.pyi

@@ -6,6 +6,7 @@
 #
 from .ten_env_attach_to_enum import _TenEnvAttachTo
 from .log_level import LogLevel
+from .addon import Addon
 
 class _Msg:
     def to_json(self) -> str: ...
@@ -179,3 +180,6 @@ class _ExtensionTester:
     def run(self) -> None: ...
 
 def _register_addon_as_extension(name: str, base_dir: str | None): ...
+def _register_addon_as_extension_v2(
+    name: str, base_dir: str | None, instance: Addon, register_ctx: object
+): ...