Merge pull request #2 from icywind/package_update

Package update

Assets/TEN/Controllers/AgoraAudioObserver.cs

@@ -0,0 +1,19 @@
+using Agora.Rtc;
+
+public class AgoraAudioObserver :  IAudioFrameObserver
+{
+    ISoundVisualizer _visual;
+    public AgoraAudioObserver(ISoundVisualizer visualizer)
+    {
+        _visual = visualizer;
+    }
+
+    public override bool OnPlaybackAudioFrameBeforeMixing(string channel_id,
+                                                    uint uid,
+                                                    AudioFrame audio_frame)
+    {
+        var floatArray = UtilFunctions.ConvertByteToFloat16(audio_frame.RawBuffer);
+        _visual?.UpdateVisualizer(floatArray);
+        return false;
+    }
+}

Assets/TEN/Controllers/IChatTextDisplay.cs

@@ -1,7 +1,5 @@
 using UnityEngine;
-using Newtonsoft.Json;
 
-using Agora.TEN.Server.Models;
 namespace Agora.TEN.Client
 {
     public abstract class IChatTextDisplay : MonoBehaviour

Assets/TEN/Controllers/ISoundVisualizer.cs

@@ -0,0 +1,8 @@
+using UnityEngine;
+using Agora.Rtc;
+
+public abstract class ISoundVisualizer : MonoBehaviour
+{
+    abstract public void UpdateVisualizer(float[] pcmData);
+    abstract public void Init(IRtcEngine RtcEngine);
+}

Assets/TEN/Controllers/SphereVisualizer.cs

@@ -1,6 +1,7 @@
 using UnityEngine;
+using Agora.Rtc;
 
-public class SphereVisualizer : MonoBehaviour
+public class SphereVisualizer : ISoundVisualizer
 {
     [SerializeField]
     float sizeMultiplier = 10f; // Multiplier for sphere size
@@ -14,6 +15,9 @@ public class SphereVisualizer : MonoBehaviour
     [SerializeField]
     bool showRotationMotion;
 
+    public int CHANNEL = 1;
+    public int SAMPLE_RATE = 44100;
+
     float[] _pcmData = new float[0];
 
     private void Update()
@@ -28,8 +32,15 @@ private void Update()
         }
     }
 
+    public override void Init(IRtcEngine RtcEngine)
+    {
+        RtcEngine.SetPlaybackAudioFrameBeforeMixingParameters(SAMPLE_RATE, CHANNEL);
+        RtcEngine.RegisterAudioFrameObserver(new AgoraAudioObserver(this),
+             AUDIO_FRAME_POSITION.AUDIO_FRAME_POSITION_BEFORE_MIXING,
+            OBSERVER_MODE.RAW_DATA);
+    }
 
-    public void UpdateVisualizer(float[] pcmData)
+    public override void UpdateVisualizer(float[] pcmData)
     {
         _pcmData = pcmData;
     }

Assets/TEN/Scenes/TENDemoChat.cs

@@ -43,8 +43,6 @@ public class TENDemoChat : MonoBehaviour
         [SerializeField]
         Button CamButton;
 
-        public int CHANNEL = 1;
-        public int SAMPLE_RATE = 44100;
 
         void Start()
         {
@@ -107,7 +105,7 @@ void SetUpUI()
 #endif
 
             // invisible until Agent joins
-            Visualizer.gameObject.SetActive(false);
+            Visualizer?.gameObject.SetActive(false);
         }
 
 
@@ -131,11 +129,7 @@ void InitEngine()
             RtcEngine.Initialize(context);
             RtcEngine.InitEventHandler(handler);
 
-            RtcEngine.SetPlaybackAudioFrameBeforeMixingParameters(SAMPLE_RATE, CHANNEL);
-
-            RtcEngine.RegisterAudioFrameObserver(new AudioFrameObserver(this),
-                 AUDIO_FRAME_POSITION.AUDIO_FRAME_POSITION_BEFORE_MIXING,
-                OBSERVER_MODE.RAW_DATA);
+            Visualizer?.Init(RtcEngine);
         }
 
         async void GetTokenAndJoin()
@@ -209,7 +203,7 @@ public override void OnUserJoined(RtcConnection connection, uint uid, int elapse
             Debug.Log(string.Format("OnUserJoined uid: ${0} elapsed: ${1}", uid,
                 elapsed));
             if (uid == AppConfig.Shared.AgentUid) {
-                _app.Visualizer.gameObject.SetActive(true);
+                _app.Visualizer?.gameObject.SetActive(true);
 			}
         }
 
@@ -219,10 +213,12 @@ public override void OnUserOffline(RtcConnection connection, uint uid, USER_OFFL
                 (int)reason));
         }
 
-        public override void OnStreamMessage(RtcConnection connection, uint remoteUid, int streamId, byte[] data, ulong length, ulong sentTs)
+        /* SDK v4.2.6 */
+        // public override void OnStreamMessage(RtcConnection connection, uint remoteUid, int streamId, byte[] data, uint length, System.UInt64 sentTs)
+        /* SDK v4.4.0 or later */
+         public override void OnStreamMessage(RtcConnection connection, uint remoteUid, int streamId, byte[] data, ulong length, ulong sentTs)
         {
             string str = System.Text.Encoding.UTF8.GetString(data, 0, (int)length);
-            // Debug.Log($"StreamMessage from:{remoteUid} ---> " + str);
             _app.TextDisplay.ProcessTextData(remoteUid, str);
             _app.TextDisplay.DisplayChatMessages(_app.LogText.gameObject);
         }
@@ -235,22 +231,6 @@ public override void OnRemoteAudioStateChanged(RtcConnection connection, uint re
 
     #endregion
 
-    internal class AudioFrameObserver : IAudioFrameObserver
-    {
-        TENDemoChat _app;
-        internal AudioFrameObserver(TENDemoChat client)
-        {
-            _app = client;
-        }
 
-        public override bool OnPlaybackAudioFrameBeforeMixing(string channel_id,
-                                                        uint uid,
-                                                        AudioFrame audio_frame)
-        {
-            var floatArray = UtilFunctions.ConvertByteToFloat16(audio_frame.RawBuffer);
-            _app.Visualizer?.UpdateVisualizer(floatArray);
-            return false;
-        }
-    }
 
 }

Assets/TEN/TENConfigInput.asset

@@ -16,9 +16,9 @@ MonoBehaviour:
   AgentUid: 1234
   AgoraAsrLanguage: en-US
   Graph: camera_va_openai_azure
-  AzureVoice: 3
+  AzureVoice: 1
   AltVoiceName: 
   ServerBaseURL: http://localhost:8080
-  AgentOpeningGreeting: TEN agent connected.  Happy to chat with you today.
+  AgentOpeningGreeting: TEN agent connected on Unity.  Happy to chat with you today.
   AgentVisionCheckingWords: '["Let me take a look...","Let me check your camera...","Please
     wait for a second..."]'

Assets/TEN/TENConfigInput.cs

@@ -28,7 +28,8 @@ public class TENConfigInput : ScriptableObject
         public AzureVoiceType AzureVoice;
 
         [SerializeField]
-        /// if it is not the provided Azure choice, enter the actual voice name here
+        [Tooltip("if the voice is not provided by Azure, enter the vendor's voice name here")]
+        /// Alternative voice override
         public string AltVoiceName;
 
         [SerializeField]

README.md

@@ -13,7 +13,7 @@ The Plugin is exported as a reusable package that can be imported on any Agora R
 ## Prerequisites:
 
 - Agora Developer account
-- Agora Video SDK for Unity (v4.4.0 or up)
+- Agora Video SDK for Unity (v4.2.6 or up)
 - [TEN Frameworks Agent](https://github.com/TEN-framework/TEN-Agent)
 - Unity 2021 or up
 
@@ -77,13 +77,13 @@ void  SetConfig()
 {
 	AppConfig.Shared.SetValue(TENConfig);
 	// obtain channel name before this call
-    AppConfig.Shared.Channel = _channelName;
+	AppConfig.Shared.Channel = UtilFunctions.GenRandomString("agora_", 5);
 }
 ```
-Note you should provide a channel name in your app logic to use among the participanting users.  In this 1-to-1 chat project, we provide an util function to ganerate a string:
-```csharp
-    _channelName = AppConfig.Shared.Channel = UtilFunctions.GenRandomString("agora_", 5);
-```
+Note you should provide a channel name in your app logic to use among the participanting users.  In this 1-to-1 chat project, we provide an util function GenRandomString() to ganerate the channel name.
+
+Don't forget to drop the TENConfigInput Asset into Inspector field for TENConfig. See demo picture in the previous section.
+
 
 2. Hook the prefabs up in your main logic:
 
@@ -105,45 +105,35 @@ async void GetTokenAndJoin()
 }
 ```
 
-4. Setup for Audio display before Mixing in InitEngine()
+4. In your initialization step, setup sound visualization, which will automatically configure the Agora RTC engine for audio data capture.
 
 ```csharp
-int  CHANNEL = 1;
-int  SAMPLE_RATE = 44100;
-RtcEngine.SetPlaybackAudioFrameBeforeMixingParameters(SAMPLE_RATE, CHANNEL);
-RtcEngine.RegisterAudioFrameObserver(new  AudioFrameObserver(this),
-AUDIO_FRAME_POSITION.AUDIO_FRAME_POSITION_BEFORE_MIXING,OBSERVER_MODE.RAW_DATA);
+	Visualizer?.Init(RtcEngine);
 ```
 
-5. Implement AudioFrameObserver:
+5. Start the TEN session on OnJoinChannelSuccess()
 ```csharp
-internal  class  AudioFrameObserver : IAudioFrameObserver
-{
-	TENDemoChat  _app;  // Replace TENDemo with your controller class name
-	internal  AudioFrameObserver(TENDemoChat  client)
-	{
-		_app = client;
-	}
-}
-
-public  override  bool  OnPlaybackAudioFrameBeforeMixing(string  channel_id, uint  uid,AudioFrame  audio_frame)
-{
-	var  floatArray = UtilFunctions.ConvertByteToFloat16(audio_frame.RawBuffer);
-	_app.Visualizer?.UpdateVisualizer(floatArray);
-	return  false;
-}
+// _app is the instance of your controller class
+_app.TENSession.StartSession(connection.localUid);
 ```
 
-
-
-
-6. Add the following to OnJoinChannelSuccess()
+6. Disable/Enable the sound visualizer, a component of a gameobject that represents the AI Agent:
+- Disable it during initialization, e.g. SetupUI() or Start()
 ```csharp
-// _app is the instance of your controller class
-_app.TENSession.StartSession(connection.localUid);
+	Visualizer?.gameObject.SetActive(false);
+```
+- Enable it when the AI Agent user joins the channel:
+```csharp
+public override void OnUserJoined(RtcConnection connection, uint uid, int elapsed)
+{
+    if (uid == AppConfig.Shared.AgentUid) {
+        _app.Visualizer?.gameObject.SetActive(true);
+	}
+}
 ```
 
 7. Register handler OnStreamMessage:
+- SDK ver 4.4.0
 ```csharp
 public  override  void  OnStreamMessage(RtcConnection  connection, uint  remoteUid, int  streamId, byte[] data, ulong  length, ulong  sentTs)
 {
@@ -153,7 +143,16 @@ public  override  void  OnStreamMessage(RtcConnection  connection, uint  remoteU
 }
 
 ```
+- SDK ver 4.2.6
+```csharp
+public override void OnStreamMessage(RtcConnection connection, uint remoteUid, int streamId, byte[] data, uint length, System.UInt64 sentTs)
+{
+    string str = System.Text.Encoding.UTF8.GetString(data, 0, (int)length);
+    _app.TextDisplay.ProcessTextData(remoteUid, str);
+    _app.TextDisplay.DisplayChatMessages(_app.LogText.gameObject);
+}
 
+```
 8. OnDestroy() or logic to stop.
 
 ```csharp