From 9f31557330341ba4e6579029ff5120c8b4ab2bfc Mon Sep 17 00:00:00 2001 From: EC <2340896+cyfyifanchen@users.noreply.github.com> Date: Thu, 29 Aug 2024 14:54:35 +0800 Subject: [PATCH] Removing ten-service to test (#2) --- .../astra-ai-agent-architecture-beta.md | 21 - ten-service/ten-api-beta.md | 1015 ----------------- ten-service/ten-architecture-beta.md | 60 - ten-service/ten-message-type-and-name-beta.md | 252 ---- ten-service/ten-schema-beta.md | 512 --------- 5 files changed, 1860 deletions(-) delete mode 100644 ten-service/astra-ai-agent-architecture-beta.md delete mode 100644 ten-service/ten-api-beta.md delete mode 100644 ten-service/ten-architecture-beta.md delete mode 100644 ten-service/ten-message-type-and-name-beta.md delete mode 100644 ten-service/ten-schema-beta.md diff --git a/ten-service/astra-ai-agent-architecture-beta.md b/ten-service/astra-ai-agent-architecture-beta.md deleted file mode 100644 index de42c6c..0000000 --- a/ten-service/astra-ai-agent-architecture-beta.md +++ /dev/null @@ -1,21 +0,0 @@ ---- -hidden: true -layout: - title: - visible: true - description: - visible: false - tableOfContents: - visible: true - outline: - visible: true - pagination: - visible: true ---- - -# 🚧 Astra AI agent architecture(beta) - -## Astra diagram - -

Astra voice agent diagram

- diff --git a/ten-service/ten-api-beta.md b/ten-service/ten-api-beta.md deleted file mode 100644 index 578ae8e..0000000 --- a/ten-service/ten-api-beta.md +++ /dev/null @@ -1,1015 +0,0 @@ ---- -hidden: true -layout: - title: - visible: true - description: - visible: false - tableOfContents: - visible: true - outline: - visible: true - pagination: - visible: true ---- - -# TEN API(beta) - -## TEN Platform API Specification - -### C Core - -#### Error - -> > Default errno, for those users only care error msgs. -> -> > Invalid json. -> -> > Invalid argument. -> -> > Invalid graph. -> -> > The TEN world is closed. - -### C++ - -#### Message - -| Name | Link | -| --------------------------------- | -------------------------------------- | -| msg\_t::get\_type | `Link ` | -| cmd\_t::get\_name | `Link ` | -| msg\_t::set\_dest | `Link ` | -| msg\_t::from\_json | `Link ` | -| msg\_t::to\_json | `Link ` | -| msg\_t::is\_property\_exist | `Link ` | -| msg\_t::get\_property\_uint8 | `Link ` | -| msg\_t::get\_property\_uint16 | `Link ` | -| msg\_t::get\_property\_uint32 | `Link ` | -| msg\_t::get\_property\_uint64 | `Link ` | -| msg\_t::get\_property\_int8 | `Link ` | -| msg\_t::get\_property\_int16 | `Link ` | -| msg\_t::get\_property\_int32 | `Link ` | -| msg\_t::get\_property\_int64 | `Link ` | -| msg\_t::get\_property\_float32 | `Link ` | -| msg\_t::get\_property\_float64 | `Link ` | -| msg\_t::get\_property\_string | `Link ` | -| msg\_t::get\_property\_bool | `Link ` | -| msg\_t::get\_property\_ptr | `Link ` | -| msg\_t::get\_property\_buf | `Link ` | -| msg\_t::get\_property\_to\_json | `Link ` | -| msg\_t::set\_property\_uint8 | `Link ` | -| msg\_t::set\_property\_uint16 | `Link ` | -| msg\_t::set\_property\_uint32 | `Link ` | -| msg\_t::set\_property\_uint64 | `Link ` | -| msg\_t::set\_property\_int8 | `Link ` | -| msg\_t::set\_property\_int16 | `Link ` | -| msg\_t::set\_property\_int32 | `Link ` | -| msg\_t::set\_property\_int64 | `Link ` | -| msg\_t::set\_property\_float32 | `Link ` | -| msg\_t::set\_property\_float64 | `Link ` | -| msg\_t::set\_property\_string | `Link ` | -| msg\_t::set\_property\_bool | `Link ` | -| msg\_t::set\_property\_ptr | `Link ` | -| msg\_t::set\_property\_buf | `Link ` | -| msg\_t::set\_property\_from\_json | `Link ` | - -C++ message API - -> Get the message type. - -> Get the message name. - -> Set the destination of the message. - -> Convert the message to JSON string. - -> Convert the message from JSON string. - -> Check if the property exists. path can not be empty. - -> Get the property from the message in the specified type. path can not be empty. - -> Get the property from the message in JSON format. path can not be empty. - -> Set the property in the message from JSON string. path can not be empty. - -#### Command - -| Name | Link | -| ----------------------------- | -------------------------------- | -| cmd\_t::create(const char \*) | `Link ` | -| cmd\_t::create\_from\_json | `Link ` | - -C++ command API - -> Create a new command with the specified command name. - -> Create a new command from a JSON string. - -#### Connect Command - -| Name | Link | -| ------------------------- | ------------------------------ | -| cmd\_connect\_t::create() | `Link ` | - -C++ connect command API - -> Create a new connect command. - -#### Status Command - -| Name | Link | -| --------------------------------------- | ------------------------------------------- | -| cmd\_result\_t::create(STATUS\_CODE) | `Link ` | -| cmd\_result\_t::get\_detail\ | `Link ` | -| cmd\_result\_t::get\_detail\_to\_json | `Link ` | -| cmd\_result\_t::set\_detail\ | `Link ` | -| cmd\_result\_t::set\_detail\_from\_json | `Link ` | -| cmd\_result\_t::get\_status\_code | `Link ` | - -C++ status command API - -> Create a new status command with the specified status code. - -> Get the detail of the status command in JSON format. - -> Set the detail of the status command from JSON string. - -> Get the status code from the status command. - -#### Timeout Command - -| Name | Link | -| --------------------------------- | ------------------------------------ | -| cmd\_timeout\_t::create() | `Link ` | -| cmd\_timeout\_t::get\_timer\_id() | `Link ` | - -C++ timeout command API - -> Create a new timeout command. - -> Get the corresponding timer ID from the timeout command. - -#### Timer Command - -| Name | Link | -| ----------------------- | ---------------------------- | -| cmd\_timer\_t::create() | `Link ` | - -C++ timer command API - -> Create a new timer command. - -#### Close App Command - -| Name | Link | -| ---------------------------- | -------------------------------- | -| cmd\_close\_app\_t::create() | `Link ` | - -C++ close app command API - -> Create a new close app command. - -#### Close Engine Command - -| Name | Link | -| ------------------------------- | ----------------------------------- | -| cmd\_close\_engine\_t::create() | `Link ` | - -C++ close engine command API - -> Create a new close engine command. - -#### Data Message - -| Name | Link | -| -------------------- | --------------------------- | -| data\_t::create() | `Link ` | -| data\_t::get\_buf | `Link ` | -| data\_t::lock\_buf | `Link ` | -| data\_t::unlock\_buf | `Link ` | - -C++ data message API - -> Create a data message. - -> Get the buffer of the data message. The operation is deep-copy. - -> Borrow the ownership of the buffer from the data message. - -> Give the ownership of the buffer back to the data message. - -#### Image frame Message - -| Name | Link | -| -------------------------------- | ------------------------------------- | -| image\_frame\_t::create() | `Link ` | -| image\_frame\_t::get\_width | `Link ` | -| image\_frame\_t::set\_width | `Link ` | -| image\_frame\_t::get\_height | `Link ` | -| image\_frame\_t::set\_height | `Link ` | -| image\_frame\_t::get\_timestamp | `Link ` | -| image\_frame\_t::set\_timestamp | `Link ` | -| image\_frame\_t::get\_pixel\_fmt | `Link ` | -| image\_frame\_t::set\_pixel\_fmt | `Link ` | -| image\_frame\_t::is\_eof | `Link ` | -| image\_frame\_t::set\_is\_eof | `Link ` | -| image\_frame\_t::alloc\_buf | `Link ` | -| image\_frame\_t::lock\_buf | `Link ` | -| image\_frame\_t::unlock\_buf | `Link ` | - -C++ image frame message API - -> Create a image frame message. - -> Get/set the width of the image frame. - -> Get/set the height of the image frame. - -> Get/set the timestamp of the image frame. - -> Get/set the pixel format type of the image frame. - -> Get/set the end of file flag of the image frame. - -> Allocate a buffer for the image frame. - -> Borrow the ownership of the buffer from the image frame message. - -> Give the ownership of the buffer back to the image frame message. - -#### Pcm frame Message - -| Name | Link | -| ----------------------------------------- | --------------------------------------------- | -| pcm\_frame\_t::create() | `Link ` | -| pcm\_frame\_t::get\_timestamp | `Link ` | -| pcm\_frame\_t::set\_timestamp | `Link ` | -| pcm\_frame\_t::get\_sample\_rate | `Link ` | -| pcm\_frame\_t::set\_sample\_rate | `Link ` | -| pcm\_frame\_t::get\_channel\_layout | `Link ` | -| pcm\_frame\_t::set\_channel\_layout | `Link ` | -| pcm\_frame\_t::get\_samples\_per\_channel | `Link ` | -| pcm\_frame\_t::set\_samples\_per\_channel | `Link ` | -| pcm\_frame\_t::get\_bytes\_per\_sample | `Link ` | -| pcm\_frame\_t::set\_bytes\_per\_sample | `Link ` | -| pcm\_frame\_t::get\_number\_of\_channels | `Link ` | -| pcm\_frame\_t::set\_number\_of\_channels | `Link ` | -| pcm\_frame\_t::get\_data\_fmt | `Link ` | -| pcm\_frame\_t::set\_data\_fmt | `Link ` | -| pcm\_frame\_t::get\_line\_size | `Link ` | -| pcm\_frame\_t::set\_line\_size | `Link ` | -| pcm\_frame\_t::is\_eof | `Link ` | -| pcm\_frame\_t::set\_is\_eof | `Link ` | -| pcm\_frame\_t::alloc\_buf | `Link ` | -| pcm\_frame\_t::lock\_buf | `Link ` | -| pcm\_frame\_t::unlock\_buf | `Link ` | - -C++ pcm frame message API - -> Create a pcm frame message. - -> Get/set the timestamp of the pcm frame. - -> Get/set the sample rate of the pcm frame. - -> Get/set the channel layout of the pcm frame. - -> Get/set the samples per channel of the pcm frame. - -> Get/set the bytes per sample of the pcm frame. - -> Get/set the number of channels of the pcm frame. - -> Get/set the data format of the pcm frame. - -> Get/set line size of the pcm frame. - -> Get/set the end of file flag of the pcm frame. - -> Allocate a buffer for the pcm frame. - -> Borrow the ownership of the buffer from the pcm frame message. - -> Give the ownership of the buffer back to the pcm frame message. - -#### Addon - -| Name | Link | -| ----------------------------------------------- | -------------------------------------------------- | -| RTE\_CPP\_REGISTER\_ADDON\_AS\_EXTENSION | `Link ` | -| RTE\_CPP\_REGISTER\_ADDON\_AS\_EXTENSION\_GROUP | `Link ` | - -C++ addon API - -> Register a C++ class as an RTE extension addon. - -> Register a C++ class as an RTE extension group addon. - -#### App - -| Name | Link | -| ---------------------------- | ------------------------- | -| app\_t::run | `Link ` | -| app\_t::close | `Link ` | -| app\_t::wait | `Link ` | -| Callback: app\_t::on\_init | `Link ` | -| Callback: app\_t::on\_deinit | `Link ` | - -C++ app API - -> Run the app. - -> Close the app. - -> Wait for the app to close. - -#### Extension - -| Name | Link | -| ------------------------------ | ------------------------------------ | -| extension\_t::on\_init | `Link ` | -| extension\_t::on\_deinit | `Link ` | -| extension\_t::on\_start | `Link ` | -| extension\_t::on\_stop | `Link ` | -| extension\_t::on\_cmd | `Link ` | -| extension\_t::on\_data | `Link ` | -| extension\_t::on\_pcm\_frame | `Link ` | -| extension\_t::on\_image\_frame | `Link ` | - -C++ extension API - -#### Extension Group - -| Name | Link | -| -------------------------------------------- | ------------------------------------------------- | -| extension\_group\_t::on\_init | `Link ` | -| extension\_group\_t::on\_deinit | `Link ` | -| extension\_group\_t::on\_create\_extensions | `Link ` | -| extension\_group\_t::on\_destroy\_extensions | `Link ` | - -C++ extension group API - -#### Metadata Info - -| Name | Link | -| ---------------------- | ----------------------------- | -| metadata\_info\_t::set | `Link ` | - -C+ + metadata info API - -> Set the metadata info. - -#### TEN Proxy - -| Name | Link | -| ---------------------------------- | --------------------------------------- | -| rte\_proxy\_t::rte\_proxy\_t | `Link ` | -| rte\_proxy\_t::acquire\_lock\_mode | `Link ` | -| rte\_proxy\_t::release\_lock\_mode | `Link ` | -| rte\_proxy\_t::notify | `Link ` | - -C++ rte proxy API - -> Create an RTE proxy instance from a RTE instance. - -> Acquire the lock mode. - -> Release the lock mode. - -> Enable the `notify_func` to be called in the RTE extension thread. - -TEN - -| Name | Link | -| ---------------------------------------- | --------------------------------------------- | -| rte\_t::send\_cmd | `Link ` | -| rte\_t::send\_json | `Link ` | -| rte\_t::send\_data | `Link ` | -| rte\_t::send\_image\_frame | `Link ` | -| rte\_t::send\_pcm\_frame | `Link ` | -| rte\_t::return\_result\_directly | `Link ` | -| rte\_t::return\_result | `Link ` | -| rte\_t::is\_property\_exist | `Link ` | -| rte\_t::get\_property\_uint8 | `Link ` | -| rte\_t::get\_property\_uint16 | `Link ` | -| rte\_t::get\_property\_uint32 | `Link ` | -| rte\_t::get\_property\_uint64 | `Link ` | -| rte\_t::get\_property\_int8 | `Link ` | -| rte\_t::get\_property\_int16 | `Link ` | -| rte\_t::get\_property\_int32 | `Link ` | -| rte\_t::get\_property\_int64 | `Link ` | -| rte\_t::get\_property\_float32 | `Link ` | -| rte\_t::get\_property\_float64 | `Link ` | -| rte\_t::get\_property\_string | `Link ` | -| rte\_t::get\_property\_bool | `Link ` | -| rte\_t::get\_property\_ptr | `Link ` | -| rte\_t::get\_property\_buf | `Link ` | -| rte\_t::get\_property\_to\_json | `Link ` | -| rte\_t::set\_property\_from\_json | `Link ` | -| rte\_t::is\_cmd\_connected | `Link ` | -| rte\_t::addon\_create\_extension\_async | `Link ` | -| rte\_t::addon\_destroy\_extension\_async | `Link ` | -| rte\_t::on\_init\_done | `Link ` | -| rte\_t::on\_deinit\_done | `Link ` | -| rte\_t::on\_start\_done | `Link ` | -| rte\_t::on\_stop\_done | `Link ` | -| rte\_t::on\_create\_extensions\_done | `Link ` | -| rte\_t::on\_destroy\_extensions\_done | `Link ` | -| rte\_t::get\_attached\_target | `Link ` | - -C++ TEN API - -> Send the cmd with a response handler. -> -> When the sending action is successful, the unique\_ptr will be released to represent that the ownership of the cmd has been transferred to the TEN runtime. Conversely, if the sending action fails, the unique\_ptr will not perform any action, indicating that the ownership of the cmd remains with the user. -> -> The type of response\_handler\_func\_t is void(rte\_t &, std::unique\_ptr\) - -> Send the command created from the json string without a response handler. - -> Send the command created from the json string with a response handler. -> -> The type of response\_handler\_func\_t is void(rte\_t &, std::unique\_ptr\) - -> Send the data message to the TEN. -> -> When the sending action is successful, the unique\_ptr will be released to represent that the ownership of the data has been transferred to the TEN runtime. Conversely, if the sending action fails, the unique\_ptr will not perform any action, indicating that the ownership of the data remains with the user. - -> Send the image frame to the TEN. -> -> When the sending action is successful, the unique\_ptr will be released to represent that the ownership of the frame has been transferred to the TEN runtime. Conversely, if the sending action fails, the unique\_ptr will not perform any action, indicating that the ownership of the frame remains with the user. - -> Send the PCM frame to the TEN. -> -> When the sending action is successful, the unique\_ptr will be released to represent that the ownership of the frame has been transferred to the TEN runtime. Conversely, if the sending action fails, the unique\_ptr will not perform any action, indicating that the ownership of the frame remains with the user. - -> Return the status command directly. -> -> When the returning action is successful, the unique\_ptr will be released to represent that the ownership of the cmd has been transferred to the RTE runtime. Conversely, if the sending action fails, the unique\_ptr will not perform any action, indicating that the ownership of the cmd remains with the user. - -> Return the status command corresponding to the target command. -> -> When the returning action is successful, the unique\_ptr will be released to represent that the ownership of the cmd has been transferred to the TEN runtime. Conversely, if the sending action fails, the unique\_ptr will not perform any action, indicating that the ownership of the cmd remains with the user. - -> Check if the property exists. path can not be empty. - -> Get the property from the TEN in JSON format. path can not be empty. - -> Set the property in the TEN from JSON string. path can not be empty. - -> Check if the command is connected in the graph. - -> Create an TEN extension instance with the specified `instance_name` from the specified addon specified with the `addon_name` asynchronously. - -> Destroy an TEN extension instance. - -> Notify the TEN that the `on_init` callback is done. - -> Notify the TEN that the `on_deinit` callback is done. - -> Notify the TEN that the `on_start` callback is done. - -> Notify the TEN that the `on_stop` callback is done. - -> Notify the TEN that the `on_create_extensions` callback is done. - -> Notify the TEN that the `on_destroy_extensions` callback is done. - -> Get the attached target. - -### Golang - -#### Message - -| Name | Link | -| ----------------------------- | ---------------------------------------- | -| msg::GetType | `Link ` | -| msg::GetName | `Link ` | -| msg::ToJSON | `Link ` | -| msg::GetPropertyInt8 | `Link ` | -| msg::GetPropertyInt16 | `Link ` | -| msg::GetPropertyInt32 | `Link ` | -| msg::GetPropertyInt64 | `Link ` | -| msg::GetPropertyUint8 | `Link ` | -| msg::GetPropertyUint16 | `Link ` | -| msg::GetPropertyUint32 | `Link ` | -| msg::GetPropertyUint64 | `Link ` | -| msg::GetPropertyBool | `Link ` | -| msg::GetPropertyPtr | `Link ` | -| msg::GetPropertyString | `Link ` | -| msg::GetPropertyBytes | `Link ` | -| msg::GetPropertyToJSONBytes | `Link ` | -| msg::SetPropertyString | `Link ` | -| msg::SetPropertyBytes | `Link ` | -| msg::SetProperty | `Link ` | -| msg::SetPropertyFromJSONBytes | `Link ` | - -Golang message API - -**GetType() MsgType** - -Get the message type. - -**GetName() (string, error)** - -Get the name of the message. - -**ToJSON() string** - -Get the JSON string of the message. - -**GetPropertyInt8(path string) (int8, error)** - -Get the property from the message in int8 type. - -**GetPropertyInt16(path string) (int16, error)** - -Get the property from the message in int16 type. - -**GetPropertyInt32(path string) (int32, error)** - -Get the property from the message in int32 type. - -**GetPropertyInt64(path string) (int64, error)** - -Get the property from the message in int64 type. - -**GetPropertyUint8(path string) (uint8, error)** - -Get the property from the message in uint8 type. - -**GetPropertyUint16(path string) (uint16, error)** - -Get the property from the message in uint16 type. - -**GetPropertyUint32(path string) (uint32, error)** - -Get the property from the message in uint32 type. - -**GetPropertyUint64(path string) (uint64, error)** - -Get the property from the message in uint64 type. - -**GetPropertyBool(path string) (bool, error)** - -Get the property from the message in bool type. - -**GetPropertyPtr(path string) (any, error)** - -Get the property from the message in ptr type. - -**GetPropertyString(path string) (string, error)** - -Get the property from the message in string type. - -**GetPropertyBytes(path string) (\[]byte, error)** - -Get the property from the message in bytes type. - -**GetPropertyToJSONBytes(path string) (\[]byte, error)** - -Get the property from the message in JSON bytes type. - -**SetPropertyString(path string, value string) error** - -Set the property in string type. - -**SetPropertyBytes(path string, value \[]byte) error** - -Set the property in bytes type. - -**SetProperty(path string, value any) error** - -Set the property. - -**SetPropertyFromJSONBytes(path string, value \[]byte) error** - -SEt the property from the JSON bytes. - -#### Command - -| Name | Link | -| ------------------- | ------------------------------------------------ | -| NewCmd | `Link ` | -| NewCmdFromJSONBytes | `Link ` | - -Golang custom command API - -**NewCmd(cmdName string) (Cmd, error)** - -Create a new custom command width the specified command name. - -**NewCmdFromJSONBytes(data \[]byte) (Cmd, error)** - -Create a new custom command from the specified JSON bytes. - -#### Status Command - -| Name | Link | -| ------------------------ | ----------------------------------- | -| NewCmdResult | `Link ` | -| CmdResult::GetStatusCode | `Link ` | - -Golang status command API - -**NewCmdResult(statusCode StatusCode, detail any) (CmdResult, error)** - -Create a new status command. - -**GetStatusCode() (StatusCode, error)** - -Get the status code from the status command. - -#### Data Message - -| Name | Link | -| ------------------------------ | ---------------------------- | -| [Data::NewData](Data::NewData) | `Link ` | -| [Data::GetBuf](Data::GetBuf) | `Link ` | - -Golang data message API - -**NewData(bytes \[]byte) (Data, error)** - -Create a new data message. - -**GetBuf() (\[]byte, error)** - -Get the data buffer from the data message. Note that this function performs a deep copy of the data buffer. - -#### Image Frame Message - -#### Pcm Frame Message - -#### Error - -> Default errno, for those users only care error msgs. - -> Invalid json. - -> Invalid argument. - -> Invalid type. - -| Name | Link | -| ---------------- | ------------------------ | -| TENError::Error | `Link ` | -| TENError::ErrNo | `Link ` | -| TENError::ErrMsg | `Link ` | - -Golang error API - -**Error() string** - -**ErrNo() uint32** - -Get the error number. - -**ErrMsg() string** - -Get the error message. - -#### Addon - -| Name | Link | -| ----------------------------- | ----------------------------------------------- | -| Addon::OnInit | `Link ` | -| Addon::OnDeinit | `Link ` | -| Addon::OnCreateInstance | `Link ` | -| RegisterAddonAsExtension | `Link ` | -| RegisterAddonAsExtensionGroup | `Link ` | -| UnloadAllAddons | `Link ` | -| NewDefaultExtensionAddon | `Link ` | -| NewDefaultExtensionGroupAddon | `Link ` | - -Golang addon API - -**OnInit(rte Rte, manifest MetadataInfo, property MetadataInfo)** - -Initialize the addon. - -**OnDeinit(rte Rte)** - -De-initialize the addon. - -**OnCreateInstance(rte Rte, name string) any** - -Create an instance of the addon. - -**RegisterAddonAsExtension(name string, addon \*Addon) error** - -Register the addon as an extension addon to the TEN runtime environment. - -**RegisterAddonAsExtensionGroup(name string, addon \*Addon) error** - -Register the addon as an extension group addon to the TEN runtime environment. - -**UnloadAllAddons() error** - -Un-register all the addons from the TEN runtime environment. - -**NewDefaultExtensionAddon(constructor func(name string) Extension) \*Addon** - -Create a new default extension addon. - -**NewDefaultExtensionGroupAddon(constructor func(name string) ExtensionGroup) \*Addon** - -Create a new default extension group addon. - -#### App - -| Name | Link | -| ------------- | ------------------------ | -| NewApp | `Link ` | -| App::OnInit | `Link ` | -| App::OnDeinit | `Link ` | -| App::Run | `Link ` | -| App::Close | `Link ` | -| App::Wait | `Link ` | - -Golang app API - -**NewApp(iApp IApp) (App, error)** - -Create a new app. - -**OnInit(rte Rte, manifest MetadataInfo, property MetadataInfo)** - -Initialize the app. - -**OnDeinit(rte Rte)** - -De-initialize the app. - -**Run(runInBackground bool)** - -Run the app. - -**Close()** - -Close the app. - -**Wait()** - -Wait the app to be closed. - -#### Extension - -| Name | Link | -| ------------------------ | ---------------------------------- | -| Extension::OnInit | `Link ` | -| Extension::OnStart | `Link ` | -| Extension::OnStop | `Link ` | -| Extension::OnDeinit | `Link ` | -| Extension::OnCmd | `Link ` | -| Extension::OnData | `Link ` | -| Extension::OnImageFrame | `Link ` | -| Extension::OnPcmFrame | `Link ` | -| Extension::WrapExtension | `Link ` | - -Golang extension API - -**OnInit(rte Rte, manifest MetadataInfo, property MetadataInfo)** - -**OnStart(rte Rte)** - -**OnStop(rte Rte)** - -**OnDeinit(rte Rte)** - -**OnCmd(rte Rte, cmd Cmd)** - -**OnData(rte Rte, data Data)** - -**OnImageFrame(rte Rte, imageFrame ImageFrame)** - -**OnPcmFrame(rte Rte, pcmFrame PcmFrame)** - -**WrapExtension(ext Extension, name string) \*extension** - -Create a new extension instance with the specified name. - -#### Extension Group - -| Name | Link | -| ----------------------------------- | ----------------------------------------------- | -| ExtensionGroup::OnInit | `Link ` | -| ExtensionGroup::OnDeinit | `Link ` | -| ExtensionGroup::OnCreateExtensions | `Link ` | -| ExtensionGroup::OnDestroyExtensions | `Link ` | -| WrapExtensionGroup | `Link ` | - -Golang extension group API - -**OnInit(rte Rte, manifest MetadataInfo, property MetadataInfo)** - -**OnDeinit(rte Rte)** - -**OnCreateExtensions(rte Rte)** - -**OnDestroyExtensions(rte Rte, extensions \[]Extension)** - -**WrapExtensionGroup(extGroup ExtensionGroup, name string) \*extensionGroup** - -Create a new extension group instance with the specified name. - -#### Metadata Info - -| Name | Link | -| ----------------- | ----------------------------- | -| MetadataInfo::Set | `Link ` | - -Golang metadata info API - -**Set(metadataType MetadataType, value string)** - -Set the metadata info. - -TEN - -| Name | Link | -| ------------------------------- | ------------------------------------------ | -| rte::GetPropertyInt8 | `Link ` | -| rte::GetPropertyInt16 | `Link ` | -| rte::GetPropertyInt32 | `Link ` | -| rte::GetPropertyInt64 | `Link ` | -| rte::GetPropertyUint8 | `Link ` | -| rte::GetPropertyUint16 | `Link ` | -| rte::GetPropertyUint32 | `Link ` | -| rte::GetPropertyUint64 | `Link ` | -| rte::GetPropertyBool | `Link ` | -| rte::GetPropertyPtr | `Link ` | -| rte::GetPropertyString | `Link ` | -| rte::GetPropertyBytes | `Link ` | -| rte::GetPropertyToJSONBytes | `Link ` | -| rte::SetPropertyString | `Link ` | -| rte::SetPropertyBytes | `Link ` | -| rte::SetProperty | `Link ` | -| rte::SetPropertyAsync | `Link ` | -| rte::SetPropertyFromJSONBytes | `Link ` | -| rte::ReturnResult | `Link ` | -| rte::ReturnResultDirectly | `Link ` | -| rte::SendJSON | `Link ` | -| rte::SendJSONBytes | `Link ` | -| rte::SendCmd | `Link ` | -| rte::SendData | `Link ` | -| rte::SendImageFrame | `Link ` | -| rte::SendPcmFrame | `Link ` | -| rte::OnStartDone | `Link ` | -| rte::OnStopDone | `Link ` | -| rte::OnInitDone | `Link ` | -| rte::OnDeinitDone | `Link ` | -| rte::OnCreateExtensionsDone | `Link ` | -| rte::OnDestroyExtensionsDone | `Link ` | -| rte::OnCreateInstanceDone | `Link ` | -| rte::IsCmdConnected | `Link ` | -| rte::AddonCreateExtensionAsync | `Link ` | -| rte::AddonDestroyExtensionAsync | `Link ` | - -Golang rte API - -**GetPropertyInt8(path string) (int8, error)** - -Get the property from the RTE in int8 type. - -**GetPropertyInt16(path string) (int16, error)** - -Get the property from the RTE in int16 type. - -**GetPropertyInt32(path string) (int32, error)** - -Get the property from the RTE in int32 type. - -**GetPropertyInt64(path string) (int64, error)** - -Get the property from the RTE in int64 type. - -**GetPropertyUint8(path string) (uint8, error)** - -Get the property from the RTE in uint8 type. - -**GetPropertyUint16(path string) (uint16, error)** - -Get the property from the RTE in uint16 type. - -**GetPropertyUint32(path string) (uint32, error)** - -Get the property from the RTE in uint32 type. - -**GetPropertyUint64(path string) (uint64, error)** - -Get the property from the RTE in uint64 type. - -**GetPropertyBool(path string) (bool, error)** - -Get the property from the RTE in bool type. - -**GetPropertyPtr(path string) (any, error)** - -Get the property from the RTE in ptr type. - -**GetPropertyString(path string) (string, error)** - -Get the property from the RTE in string type. - -**GetPropertyBytes(path string) (\[]byte, error)** - -Get the property from the RTE in bytes type. - -**GetPropertyToJSONBytes(path string) (\[]byte, error)** - -Get the property from the RTE in JSON bytes type. - -**SetProperty(path string, value any) error** - -Set the property to the RTE. - -**SetPropertyString(path string, value string) error** - -Set the property to the RTE in string type. - -**SetPropertyBytes(path string, value \[]byte) error** - -Set the property to the RTE in bytes type. - -**SetPropertyFromJSONBytes(path string, value \[]byte) error** - -Set the property to the RTE from the JSON bytes. - -**SetPropertyAsync(path string, v any, callback func(Rte, error)) error** - -Set the property to the RTE asynchronously. - -**ReturnResult(statusCmd CmdResult, cmd Cmd) error** - -Return the result. - -**ReturnResultDirectly(statusCmd CmdResult) error** - -Return the result directly. - -**SendJSON(json string, handler ResponseHandler) error** - -Send the JSON. - -**SendJSONBytes(json \[]byte, handler ResponseHandler) error** - -Send the JSON bytes. - -**SendCmd(cmd Cmd, handler ResponseHandler) error** - -Send the command. - -**SendData(data Data) error** - -Send the data. - -**SendImageFrame(imageFrame ImageFrame) error** - -Send the image frame. - -**SendPcmFrame(pcmFrame PcmFrame) error** - -Send the PCM frame. - -**OnInitDone(manifest MetadataInfo, property MetadataInfo) error** - -OnInit done. - -**OnStartDone() error** - -OnStart done. - -**OnStopDone() error** - -OnStop done. - -**OnDeinitDone() error** - -OnDeinit done. - -**OnCreateExtensionsDone(extensions ...Extension) error** - -OnCreateExtensions done. - -**OnDestroyExtensionsDone() error** - -OnDestroyExtensions done. - -**OnCreateInstanceDone(instance any) error** - -OnCreateInstance done. - -**IsCmdConnected(cmdName string) (bool, error)** - -Is the command connected. - -**AddonCreateExtensionAsync(addonName string, instanceName string, callback func(rte Rte, p Extension)) error** - -Create an extension from the addon specified by the addon name and instance name. - -**AddonDestroyExtensionAsync(ext Extension, callback func(rte Rte)) error** - -Destroy a specified extension. diff --git a/ten-service/ten-architecture-beta.md b/ten-service/ten-architecture-beta.md deleted file mode 100644 index 580e8d4..0000000 --- a/ten-service/ten-architecture-beta.md +++ /dev/null @@ -1,60 +0,0 @@ ---- -layout: - title: - visible: true - description: - visible: false - tableOfContents: - visible: true - outline: - visible: true - pagination: - visible: true ---- - -# 🚧 TEN architecture(beta) - -## TEN architecture - -Now let's discuss what's under the hood. The TEN architecture is composed of various TEN extensions, developed in different programming languages. These extensions are interconnected using Graph, which describes their relationships and illustrates the flow of data. Furthermore, sharing and downloading extensions are simplified through the TEN Extension Store and the TEN Package Manager. - -## TEN extension - -An extension is the fundamental unit of composition within the TEN framework. Developers can create extensions in various programming languages and combine them to build diverse scenarios and applications. TEN emphasizes cross-language collaboration, allowing extensions written in different languages to work together seamlessly within the same application or service. - -For example, if an application requires real-time communication (RTC) features and advanced AI capabilities, a developer might choose to write RTC-related extensions in C++ for its performance advantages in processing audio and video data. Meanwhile, they could develop AI extensions in Python to leverage its extensive libraries and frameworks for data analysis and machine learning tasks. - - - -## TEN graph - -A Graph in TEN describes the data flow between extensions, orchestrating their interactions. For example, the text output from a speech-to-text (STT) extension might be directed to a large language model (LLM) extension. Essentially, a Graph defines which extensions are involved and the direction of data flow between them. Developers can customize this flow, directing outputs from one extension, such as an STT, into another, like an LLM. - -In TEN, there are four main types of data flow between extensions, they are **Command**, **Data**, **Image Frame** and **PCM Frame**. - -By specifying the direction of these data types in the Graph, developers can enable mutual invocation and unidirectional data flow between plugins. This is especially useful for PCM and image data types, simplifying audio and video processing. - - - -## TEN agent app - -A TEN Agent App is a runnable server-side application that combines multiple Extensions following Graph rules to accomplish more sophisticated operations. A TEN Agent App is a robust, server-side application that executes complex operations by integrating multiple Extensions within a flexible framework defined by Graph rules. These Graph rules orchestrate the interplay between various Extensions, enabling the app to perform sophisticated tasks that go beyond the capabilities of individual components. - -By leveraging this architecture, a TEN Agent App can seamlessly manage and coordinate different functionalities, ensuring that each Extension interacts harmoniously with others. This design allows developers to create powerful and scalable applications capable of handling intricate workflows and data processing requirements. - - - -## TEN extension store - -The TEN Store is a centralized platform designed to foster collaboration and innovation among developers by providing a space where they can share their extensions. This allows developers to contribute to the community, showcase their work, and receive feedback from peers, enhancing the overall quality and functionality of the TEN ecosystem. - -In addition to sharing their own extensions, developers can also access a wide array of extensions created by others. This extensive library of extensions makes it easier to find tools and functionalities that can be integrated into their own projects, accelerating development and promoting best practices within the community. The TEN Store thus serves as a valuable resource for both novice and experienced developers looking to expand their capabilities and leverage the collective expertise of the community. - - - -## TEN package manager - -The TEN Package Manager streamlines the entire process of handling TEN extensions, making it easy to upload, share, download, and install them. It significantly simplifies the workflow by allowing extensions to specify their dependencies on other extensions and the environment. This ensures that all necessary components are automatically managed and installed, reducing the potential for errors and conflicts. - -By automatically managing these dependencies, the TEN Package Manager makes the installation and release of extensions extremely convenient and intuitive. This tool not only saves time but also enhances the user experience by ensuring that every extension works seamlessly within the larger ecosystem. This level of automation and ease of use encourages the development and distribution of more robust and complex extensions, further enriching the TEN framework. - diff --git a/ten-service/ten-message-type-and-name-beta.md b/ten-service/ten-message-type-and-name-beta.md deleted file mode 100644 index 7cd2534..0000000 --- a/ten-service/ten-message-type-and-name-beta.md +++ /dev/null @@ -1,252 +0,0 @@ ---- -layout: - title: - visible: true - description: - visible: false - tableOfContents: - visible: true - outline: - visible: true - pagination: - visible: true ---- - -# 🚧 TEN Message type and name(beta) - -## Message Type and Name - -Below is an overview diagram of an TEN platform message. - -``` -┌── has response -│ └── command -│ ├── TEN platform built-in command -│ │ => message names start with `ten::` -│ └── Non-TEN platform built-in command -│ => message names do not start with `ten::` -└── no response - ├── data - │ ├── TEN platform built-in data - │ │ => message names start with `ten::` - │ └── Non-TEN platform built-in data - │ => message names do not start with `ten::` - ├── image_frame - │ ├── TEN platform built-in image_frame - │ │ => message names start with `ten::` - │ └── Non-TEN platform built-in image_frame - │ => message names do not start with `ten::` - └── pcm_frame - ├── TEN platform built-in pcm_frame - │ => message names start with `ten::` - └── Non-TEN platform built-in pcm_frame - => message names do not start with `ten::` -``` - -### Differentiating Message Type and Message Name - -When different messages have different functionalities, message types are used to differentiate them. Functionalities refer to what the TEN Platform provides for a message, and one of the functionalities is an API. For example, the TEN Platform provides an API for getting/setting image frame data. Other functionalities include whether the message has a response, etc. For instance: - -* If a message has a response (referred to as a status command in the TEN Platform), there will be a message type representing this type of message. -* When a message has an image buffer (YUV422, RGB, etc.) that can be accessed, there will be a message type representing this message with the field. This message type provides the API for getting/setting image frame data. -* When a message has an audio buffer that can be accessed, a message type called pcm frame is created. This message type provides the API for getting/setting pcm frame data. - -Message names are used to differentiate the different purposes of messages within the same message type. - -### Message Type - -The TEN Platform messages have four types: - -1. command -2. data -3. image frame -4. pcm frame - -The difference between commands and non-commands is that commands have a response (referred to as a status command in the TEN Platform), while non-commands do not have a response. - -The corresponding extension message callbacks are: - -1. OnCmd -2. OnData -3. OnImageFrame -4. OnPcmFrame - -Although there are currently four message types, it is not certain if there will only be these four types in the future. There may be new types added. If we consider this, merging the four extension message callbacks into one OnMsg can avoid the issue of adding a new extension message callback for each new message type. However, this may inconvenience users, as it means they would have to handle different message types within the OnMsg function (pseudo code). - -> ```c++ -> OnMsg(msg) { -> switch (msg->type) { -> case command || connect || timer || ...: -> OnCmd(msg); -> break; -> case data: -> OnData(msg); -> break; -> case image_frame: -> OnImageFrame(msg); -> break; -> case pcm_frame: -> OnPcmFrame(msg); -> break; -> } -> } -> ``` - -Additionally, in the TEN graph, there is a distinction between cmd\_in, cmd\_out, data\_in, data\_out, etc. So, following a unified approach, the interface of the extension should still maintain the differentiation of different message types. - -Note - -If no message type is specified, the default type is cmd. - -### Message Name - -Message Name is used within the TEN runtime to differentiate messages with different purposes under the same message type. The extension determines what actions to take based on the differentiation of message names. - -The naming convention for message names is as follows: - -1. The first character can only be a-z, A-Z, or \_. -2. Other characters can be a-z, A-Z, 0-9, or \_. - -### Creating Messages - -Non-TEN platform built-in command: - -```json -{ - "ten": { - "type": "cmd", // mandatory - "name": "hello_world" // mandatory - } -} -``` - -TEN platform built-in command: - -```json -{ - "ten": { - "type": "cmd", // mandatory - "name": "ten::connect" // mandatory - } -} -``` - -Data: - -```json -{ - "ten": { - "type": "data", // mandatory - "name": "foo" // optional - } -} -``` - -Image Frame: - -```json -{ - "ten": { - "type": "image_frame", // mandatory - "name": "foo" // optional - } -} -``` - -PCM Frame: - -```json -{ - "ten": { - "type": "pcm_frame", // mandatory - "name": "foo" // optional - } -} -``` - -### An Optimization in TEN Runtime for Message Names - -Within the TEN runtime, each message is recorded with two fields: - -1. Message type - - Enum. -2. Message name - - String. - -Since message names are in string format, there is a certain performance overhead when comparing strings. Therefore, when the TEN runtime encounters certain specific message names, it can use a message type to represent that message. For example, when it sees a message name like ten::connect, the TEN runtime can optimize it from: - -* Message type: cmd // mandatory -* Message name: ten::connect // mandatory - -to: - -* Message type: connect // mandatory -* Message name: ten::connect // optional - -This optimization can only be applied to message names recognized by the TEN platform, so it is only possible for TEN platform built-in message names. Users cannot perform this optimization themselves, and new message types cannot be added by users. Since built-in messages are limited, the newly added message types are also limited, aligning with the enum type of message types. - -In summary, message types other than cmd, data, image\_frame, and pcm\_frame are generated through this optimization, and these additional message types correspond to a special message name, which is always an TEN built-in message. - -This optimization is not only applicable to commands but also to other message types. For example: - -* Message type: image\_frame // mandatory -* Message name: ten::empty\_image\_frame // mandatory - -can be optimized to: - -* Message type: image\_frame\_empty // mandatory -* Message name: ten::empty\_image\_frame // optional - -Since the TEN platform modifies the type field of the message, users can also see and use this optimization. This is not a bad thing as users can leverage it to speed up message analysis. Therefore, users can use two methods to determine the message: - -1. Using the message name - - ```c++ - if (message_name == "ten::timer") - ``` -2. Using the message type - - ```c++ - if (message_type == MSG_TYPE_TIMER) - ``` - -### Creating Messages (with the mentioned optimization) - -TEN platform 的 built-in command: - -```json -{ - "ten": { - "type": "cmd", // mandatory - "name": "ten::connect" // mandatory - } -} -``` - -Or - -```json -{ - "ten": { - "type": "connect" // mandatory - } -} -``` - -### When to Specify Message Type or Message Name - -When it is not possible to determine the message type or message name in the context, it is necessary to specify the message type or message name. - -For example: - -* Message from JSON - - When the message type or message name is unknown, it is necessary to specify the message type and message name in the JSON. -* Command from JSON - - When it is known that the message is a command but the command name is unknown, it is necessary to specify the message name in the JSON. -* Connect command from JSON - - When it is known that the message is a connect command, the message name can only be ten::connect. Therefore, there is no need to specify the message type or message name in the JSON. diff --git a/ten-service/ten-schema-beta.md b/ten-service/ten-schema-beta.md deleted file mode 100644 index a90d9b3..0000000 --- a/ten-service/ten-schema-beta.md +++ /dev/null @@ -1,512 +0,0 @@ ---- -layout: - title: - visible: true - description: - visible: false - tableOfContents: - visible: true - outline: - visible: true - pagination: - visible: true ---- - -# 🚧 TEN schema(beta) - -## Overview - -TEN Schema provides a way to describe the data structure of TEN Value. TEN Value is a structured data type used in TEN Runtime to store information such as properties. TEN Schema can be used to define the data structure of configuration files for Extensions, as well as the data structure of messages sent and received by Extensions. - -The most common use case is when there are two Extensions - A and B, and A sends a command to B. In this scenario, TEN Schema is involved in the following places: - -* Configuration file of Extension A. -* Configuration file of Extension B. -* Data structure of the message carried by the command sent by A as a producer. -* Data structure of the message carried by the command received by B as a consumer. - -### Configuration Files - -RTE Extensions typically have two configuration files: - -`property.json`: This file contains the business-specific configuration of the Extension, such as: - -{% code title="property.josn" %} -```json -{ - "app_id": "123456", - "channel": "test", - "log": { - "level": 1, - "redirect_stdout": true, - "file": "api.log" - } -} -``` -{% endcode %} - -`manifest.json`: This file contains the immutable information of the Extension, including metadata (type, name, version, etc.) and schema definitions. For example: - -{% code title="manifest.json" %} -```json -{ - "type": "extension", - "name": "A", - "version": "1.0.0", - "language": "cpp", - "dependencies": [], - "api": {} -} -``` -{% endcode %} - -Developers can customize the logic for loading configuration files in the Extension `on_init()` function. By default, TEN Extension automatically loads the `property.json` and `manifest.json` files in the directory. After the Extension`:on_start()` callback is triggered, developers can use rte:`get_property()` to retrieve the contents of `property.json`. For example: - -````cpp -``` -void on_init(rte::rte_t &rte, rte::metadata_info_t &manifest, - rte::metadata_info_t &property) override { - // You can load custom configuration files using the following method. In this case, the default property.json will not be loaded. - // property.set(RTE_METADATA_JSON_FILENAME, "a.json"); - - rte.on_init_done(manifest, property); -} -```` - -After the `rte.on_init_done()` is triggered, the content of property.json will be stored in TEN Runtime as an TEN Value, with the type of Object. Here are some details: - -* `app_id` is a Value of type String, and developers can retrieve it using `rte.get_property_string("app_id")`, specifying the type as string. -* `log` is a Value of type Object. In the TEN Value system, Object is a composite structured data that contains other TEN Values in key-value pairs. For example, `level` is a field in `log` and its type is int64. - - - -{% hint style="info" %} -Note - -This involves one of the purposes of TEN Schema: to explicitly declare the precision of Values. - -For JSON, an integer value like `1` is by default parsed as `int64` in x64. When TEN Runtime reads JSON, it can only handle it as `int64` by default. Even if the developer expects the type to be `uint8`, this information cannot be inferred from JSON alone. This is where TEN Schema comes in, to explicitly declare the type of `level` as `uint8`. After parsing property.json with TEN Runtime, the value of `level` will be evaluated based on the definition in the TEN Schema. If it meets the storage requirements of `uint8`, it will be stored as `uint8` in the TEN Value. - -Note - -One of the rules followed by TEN Schema is to ensure data integrity and no loss of precision during type conversion. It does not support cross-type re-parsing, such as converting int to double or string. -{% endhint %} - - - -### TEN Schema - -The type in TEN Schema is exactly the same as the type in TEN Value, including: - -
integerunsigned integerfloat pointother
int8uint8float32string
int16uint16float64bool
int32uint32buf
int64uint64ptr
array
object
- - - -Except for the simple data types (array and object), the declaration is as follows: - -```json -{ - "type": "int8" -} -``` - -For the array type, the declaration is as follows: - -```json -{ - "type": "array", - "items": { - "type": "int8" - } -} -``` - -For the object type, the declaration is as follows: - -```json -{ - "type": "object", - "properties": { - "field_1": { - "type": "int8" - }, - "field_2": { - "type": "bool" - } - } -} -``` - -Using the example of property.json mentioned above, its corresponding TEN Schema is as follows: - -```json -{ - "type": "object", - "properties": { - "app_id": { - "type": "string" - }, - "channel": { - "type": "string" - }, - "log": { - "type": "object", - "properties": { - "level": { - "type": "uint8" - }, - "redirect_stdout": { - "type": "bool" - }, - "file": { - "type": "string" - } - } - } - } -} -``` - -Note - -* Currently, TEN Schema only supports the type keyword to meet the basic needs. It will be gradually improved based on future requirements. - -### manifest.json - -The definition of the TEN Schema for the Extension needs to be saved in the manifest.json file. Similar to property.json, after the rte.on\_init\_done() is executed, manifest.json will be parsed by the TEN Runtime. This allows the TEN Runtime to validate the data integrity based on the schema definitions when loading property.json. - -The TEN Schema is placed under the api section in manifest.json. The api section is a JSON object that contains the definitions of all the schemas involved in the Extension, including the configurations mentioned above and the messages (TEN cmd/data/image\_frame/pcm\_frame) that will be introduced next. The structure of the api section is as follows: - -{% code title="manifest.json" %} -```json -{ - "property": {}, - "cmd_in": [], - "cmd_out": [], - "data_in": [], - "data_out": [], - "image_frame_in": [], - "image_frame_out": [], - "pcm_frame_in": [], - "pcm_frame_out": [], - "interface_in": [], - "interface_out": [] -} -``` -{% endcode %} - -The schema for property.json is placed under the property section. The content of the manifest.json corresponding to the example mentioned above should be as follows: - -{% code title="property.json" %} -```json -{ - "type": "extension", - "name": "A", - "version": "1.0.0", - "language": "cpp", - "dependencies": [], - "api": { - "property": { - "app_id": { - "type": "string" - }, - "channel": { - "type": "string" - }, - "log": { - "type": "object", - "properties": { - "level": { - "type": "uint8" - }, - "redirect_stdout": { - "type": "bool" - }, - "file": { - "type": "string" - } - } - } - } - } -} -``` -{% endcode %} - -### Messages - -Extensions can exchange messages, including cmd/data/image\_frame/pcm\_frame. If it is used as a producer (e.g., calling rte.send\_cmd()), the corresponding schema is defined in the xxx\_in section of manifest.json, such as cmd\_in. Similarly, if it is used as a consumer (e.g., receiving messages in the on\_cmd() callback), the corresponding schema is defined in the xxx\_out section of manifest.json, such as cmd\_out. - -The schema definition for messages is indexed by name. - -cmd must specify the name. For example: - -```json -{ - "api": { - "cmd_in": [ - { - "name": "start", - "property": { - "app_id": { - "type": "string" - } - } - } - ] - } -} -``` - -For data/image\_frame/pcm\_frame, the name is optional. This means that the name in the schema definition is optional; schemas without a specified name will be considered as the default schema. In other words, for any message, if a name exists, the schema will be indexed by that name; if not found, the default schema will be used. For example: - -```json -{ - "api": { - "data_in": [ - { - "property": { - "width": { - "type": "uint32" - } - } - }, - { - "name": "speech", - "property": { - "width": { - "type": "uint16" - } - } - } - ] - } -} -``` - -Similarly, when using JSON as the property of a message, if a schema exists, the types and precision will be converted according to the definitions in the schema. - -For the example mentioned above, Extension A's cmd\_out (i.e., A calling rte.send\_cmd()) corresponds to Extension B's cmd\_in (i.e., B receiving the message in the on\_cmd() callback). - -Note - -* Currently, if the cmd sent by A does not comply with the schema definition, B will not receive the cmd, and the TEN Runtime will return an error to A. - -Compared to data/image\_frame/pcm\_frame, cmd has an ack (represented as status cmd in TEN). This means that when defining the cmd schema, you can also define the schema for the corresponding status cmd. For example: - -```json -{ - "api": { - "cmd_in": [ - { - "name": "start", - "property": { - "app_id": { - "type": "string" - } - }, - "result": { - "property": { - "detail": { - "type": "string" - }, - "code": { - "type": "uint8" - } - } - } - } - ] - } -} -``` - -Note - -* "status" is the annotation key used to define the status schema. -* "rte/detail" is the annotation key used to define the type of detail in the status. It is used in APIs like rte.return\_string(), cmd.get\_detail(), etc. -* "status/property" is the annotation key that works the same as the property in cmd. -* The "status" in cmd\_in refers to the response received from the downstream Extension as a producer. Similarly, the "status" in cmd\_out refers to the response sent back to the upstream as a consumer. - -### Example - -Taking the rte\_stt\_asr\_filter extension as an example, let's see how to define the schema. - -First, the property.json file under the rte\_stt\_asr\_filter extension is not used. Its configuration is specified in the Graph as follows: - -```json -{ - "type": "extension", - "name": "rte_stt_asr_filter", - "addon": "rte_stt_asr_filter", - "extension_group": "rtc_group", - "property": { - "app_id": "1a4dcbc3", - "api_key": "b6e21445580c80a2a62a9c0394bc5e83", - "api_secret": "ZTliNzFhODU2MDMzYzQzYzUxODNmY2Ix", - "plugin_path": "addon/extension/rte_stt_asr_filter/lib/liblinux_audio_hy_extension.so", - "data_encoding": "raw" - } -} -``` - -The corresponding schema should be: - -```json -{ - "api": { - "property": { - "app_id": { - "type": "string" - }, - "api_key": { - "type": "string" - }, - "api_secret": { - "type": "string" - }, - "plugin_path": { - "type": "string" - }, - "data_encoding": { - "type": "string" - } - } - } -} -``` - -Next, it will receive the following five cmds: - -```cpp -if (command == COMMAND_START) { - handleStart(rte, std::move(cmd)); -} else if (command == COMMAND_STOP) { - handleStop(rte, std::move(cmd)); -} else if (command == COMMAND_ON_USER_AUDIO_TRACK_SUBSCRIBED) { - handleUserAudioTrackSubscribed(rte, std::move(cmd)); -} else if (command == COMMAND_ON_USER_AUDIO_TRACK_STATE_CHANGED) { - handleUserAudioTrackStateChanged(rte, std::move(cmd)); -} else if (command == COMMAND_QUERY) { - handleQuery(rte, std::move(cmd)); -} else { - RTE_LOGE("FATAL: unknown command: %s", command.c_str()); - rte.return_string(RTE_STATUS_CODE_ERROR, "not implemented", std::move(cmd)); -} -``` - -Let's take start and onUserAudioTrackSubscribed cmds as examples. - -start - -```cpp -class Config -{ - // ... - - private: - std::vector languages_; - std::string licenseFilePath_; -}; - -void from_json(const nlohmann::json& j, Config& p) -{ - try { - j.at("languages").get_to(p.languages_); - } catch (std::exception& e) { - RTE_LOGW("Failed to parse 'languages' property: %s", e.what()); - } - - try { - j.at("licenseFilePath").get_to(p.licenseFilePath_); - } catch (std::exception& e) { - RTE_LOGW("Failed to parse 'licenseFilePath' property: %s", e.what()); - } -} -``` - -For the start cmd, it includes two properties: - -* `languages`: an array type with elements of string type. -* `licenseFilePath`: a string type. - -So, the corresponding schema should be: - -```json -{ - "api": { - "cmd_in": [ - { - "name": "start", - "property": { - "languages": { - "type": "array", - "items": { - "type": "string" - } - }, - "licenseFilePath": { - "type": "string" - } - }, - "result": { - "property": { - "detail": { - "type": "string" - } - } - } - } - ] - } -} -``` - -onUserAudioTrackSubscribed - -```cpp -void rte_stt_asr_filter_extension_t::handleUserAudioTrackSubscribed( - rte::rte_t &rte, std::unique_ptr cmd) { - auto user_id = cmd->get_property_string("userId"); - auto *track = - cmd->get_property("audioTrack"); - - // ... - - rte.return_string(RTE_STATUS_CODE_OK, "done", std::move(cmd)); -} -``` - -For the onUserAudioTrackSubscribed cmd, it includes two properties: - -* `userId`: a string type. -* `audioTrack`: a ptr type pointing to agora::rtc::IRemoteAudioTrack. - -So, the corresponding schema should be: - -```json -{ - "api": { - "cmd_in": [ - { - "name": "onUserAudioTrackSubscribed", - "property": { - "userId": { - "type": "string" - }, - "audioTrack": { - "type": "ptr" - } - }, - "result": { - "property": { - "detail": { - "type": "string" - } - } - } - } - ] - } -} -```