Implemented send / recv MetadataFrame, recv_send_metadata and send_ca…

…pture

Packages/jp.keijiro.klak.ndi/Runtime/Component/NdiReceiver.cs

@@ -13,6 +13,7 @@ public sealed partial class NdiReceiver : MonoBehaviour
 
     List<Interop.VideoFrame> videoFrameQueue = new List<Interop.VideoFrame>();
     List<float[]> audioBufferQueue = new List<float[]>();
+    List<string> metadataQueue = new List<string>();
 
     Thread _receiveThread;
     object threadlock = new object();
@@ -71,15 +72,32 @@ void TryReceiveFrame()
                     // We cannot free up the frame here because we need the data
                     // IntPtr address to be valid when processing on main thread.
                     // So we have to make sure to free the videoframe after processing
-                    lock (threadlock)
+                    lock (videoFrameQueue)
                     {
                         // We only need to keep a single frame at any point
                         // So we can free up any previous frames first
                         videoFrameQueue.ForEach(vf => _recv.FreeVideoFrame(vf));
                         videoFrameQueue.Clear();
                         videoFrameQueue.Add(captureFrame.videoFrame);
-                    }
-                    break;
+
+                            // Send some metadata back
+                            if (!string.IsNullOrEmpty(sendMetadataFrameData))
+                            {
+                                Interop.MetadataFrame metadataFrame = new Interop.MetadataFrame();
+                                int length;
+                                metadataFrame.Data = Util.StringToUtf8(sendMetadataFrameData, out length);
+                                metadataFrame.Length = length;
+                                if (_recv != null && !_recv.IsClosed)
+                                {
+                                    _recv.SendMetadata(metadataFrame);
+                                }
+                                Marshal.FreeHGlobal(metadataFrame.Data);
+                            }
+
+                        }
+
+
+                        break;
 
                 case Interop.FrameType.Audio:
 
@@ -117,7 +135,7 @@ void TryReceiveFrame()
                     handle.Free();
 
                     // Add buffer to queue for processing in audio thread
-                    lock (threadlock)
+                    lock (audioBufferQueue)
                     {
                         audioBufferQueue.Add(Util.ConvertByteArrayToFloat(audBuffer));
                     }
@@ -127,8 +145,22 @@ void TryReceiveFrame()
                     _recv.FreeAudioFrame(captureFrame.audioFrame);
 
                     break;
+
+                case Interop.FrameType.Metadata:
+
+                    // UTF-8 strings must be converted for use - length includes the terminating zero
+                    string metadata = Util.Utf8ToString(captureFrame.metadataFrame.Data, (uint)captureFrame.metadataFrame.Length-1);
+
+                    lock(metadataQueue)
+                    {
+                        metadataQueue.Add(metadata);
+                    }
+
+                    // free frames that were received
+                    _recv.FreeMetadataFrame(captureFrame.metadataFrame);
+                    break;
+                }
             }
-        }
     }
 
     #endregion
@@ -153,7 +185,7 @@ void Start()
 
     private void OnAudioFilterRead(float[] data, int channels)
     {
-        lock (threadlock)
+        lock (audioBufferQueue)
         {
             if (audioBufferQueue.Count > 0)
             {
@@ -174,8 +206,25 @@ void LateUpdate()
         if (_recv == null) return;
 
         RenderTexture rt = null;
-        lock (threadlock)
-        {
+            // Handle MetadataFrames
+            string metadataFrameData = null;
+
+            lock (metadataQueue)
+            {
+                if (metadataQueue.Count > 0)
+                {
+                    metadataFrameData = metadataQueue[0];
+                    metadataQueue.RemoveAt(0);
+                }
+            }
+            // Do something with your metadataFrameData here
+            if (!string.IsNullOrEmpty(metadataFrameData))
+            { 
+                Debug.Log(metadataFrameData);
+            }
+
+            lock(videoFrameQueue)
+            { 
             // Handle VideoFrames                
             if (videoFrameQueue.Count > 0)
             {
@@ -191,9 +240,9 @@ void LateUpdate()
 
                 // Copy the metadata if any.
                 if (vf.Metadata != IntPtr.Zero)
-                    metadata = Marshal.PtrToStringAnsi(vf.Metadata);
+                    recvVideoFrameMetadata = Marshal.PtrToStringAnsi(vf.Metadata);
                 else
-                    metadata = null;
+                    recvVideoFrameMetadata = null;
 
                 // Store the videoframe resolution
                 if (resolution == null || resolution.x != vf.Width || resolution.y != vf.Height)

Packages/jp.keijiro.klak.ndi/Runtime/Component/NdiReceiverProperties.cs

@@ -46,7 +46,9 @@ public AudioSource targetAudioSource
 
     public RenderTexture texture => _converter.LastDecoderOutput;
 
-    public string metadata { get; set; }
+    public string recvVideoFrameMetadata { get; set; }
+
+    public string sendMetadataFrameData { get; set; }
 
     public Interop.Recv internalRecvObject => _recv;
 

Packages/jp.keijiro.klak.ndi/Runtime/Component/NdiSender.cs

@@ -1,3 +1,4 @@
+using System.Runtime.InteropServices;
 using Unity.Collections.LowLevel.Unsafe;
 using UnityEngine;
 using UnityEngine.Rendering;
@@ -91,11 +92,11 @@ System.Collections.IEnumerator ImmediateCaptureCoroutine()
             if (converted == null) continue;
 
             AsyncGPUReadback.Request(converted, _onReadback);
-            _metadataQueue.Enqueue(metadata);
+            _metadataQueue.Enqueue(sendVideoFrameMetadata);
         }
     }
 
-    #endregion
+        #endregion
 
     #region SRP camera capture callback
 
@@ -117,7 +118,7 @@ void OnCameraCapture(RenderTargetIdentifier source, CommandBuffer cb)
 
         // GPU readback request
         cb.RequestAsyncReadback(converted, _onReadback);
-        _metadataQueue.Enqueue(metadata);
+        _metadataQueue.Enqueue(sendVideoFrameMetadata);
     }
 
     #endregion
@@ -157,6 +158,17 @@ unsafe void OnReadback(AsyncGPUReadbackRequest request)
 
             // Send via NDI
             _send.SendVideoAsync(frame);
+
+            // Check if the receiver has returned any metadataFrames
+            Interop.MetadataFrame metadataFrame = new Interop.MetadataFrame();
+            while (_send.Capture(out metadataFrame, 0) == Interop.FrameType.Metadata)
+            {
+                // Store the latest 
+                recvMetadataFrameData = Util.Utf8ToString(metadataFrame.Data);
+
+                // Free the metadataFrame
+                _send.FreeMetadata(ref metadataFrame);
+            }
         }
     }
 
@@ -285,6 +297,23 @@ void OnAudioFilterRead(float[] data, int channels)
         }
     }
 
+    private void LateUpdate()
+    {
+        if (!string.IsNullOrEmpty(sendMetadataFrameData))
+        {
+            // Send some metadata
+            Interop.MetadataFrame metadataFrame = new Interop.MetadataFrame();
+            int length;
+            metadataFrame.Data = Util.StringToUtf8(sendMetadataFrameData, out length);
+            metadataFrame.Length = length;
+            if (_send != null && !_send.IsClosed)
+            {
+                _send.SendMetadata(metadataFrame);
+            }
+            Marshal.FreeHGlobal(metadataFrame.Data);
+            sendMetadataFrameData = null;
+        }
+    }
     #endregion
 }
 

Packages/jp.keijiro.klak.ndi/Runtime/Component/NdiSenderProperties.cs

@@ -46,7 +46,12 @@ public Texture sourceTexture
 
     #region Runtime property
 
-    public string metadata { get; set; }
+    public string sendVideoFrameMetadata { get; set; }
+
+    public string sendMetadataFrameData { get; set; }
+
+    public string recvMetadataFrameData { get; set; }
+
 
     public Interop.Send internalSendObject => _send;
 

Packages/jp.keijiro.klak.ndi/Runtime/Internal/RecvHelper.cs

@@ -32,18 +32,18 @@ public static unsafe Interop.Recv TryCreateRecv(string sourceName)
         return (Interop.VideoFrame?)video;
     }
 
-        public static Interop.CaptureFrame TryCaptureFrame(Interop.Recv recv)
-        {
-            Interop.CaptureFrame captureFrame = new Interop.CaptureFrame();
-
-            captureFrame.frameType = recv.Capture(
-                out captureFrame.videoFrame,
-                out captureFrame.audioFrame,
-                IntPtr.Zero,
-                0);
-
-            return captureFrame;
-        }
+    public static Interop.CaptureFrame TryCaptureFrame(Interop.Recv recv)
+    {
+        Interop.CaptureFrame captureFrame = new Interop.CaptureFrame();
+
+        captureFrame.frameType = recv.Capture(
+            out captureFrame.videoFrame,
+            out captureFrame.audioFrame,
+            out captureFrame.metadataFrame,
+        0);
+
+        return captureFrame;
+    }
 }
 
 }

Packages/jp.keijiro.klak.ndi/Runtime/Internal/Utility.cs

@@ -1,3 +1,6 @@
+using System;
+using System.Runtime.InteropServices;
+using System.Text;
 using UnityEngine;
 using BindingFlags = System.Reflection.BindingFlags;
 using Delegate = System.Delegate;
@@ -25,6 +28,68 @@ public static float[] ConvertByteArrayToFloat(byte[] bytes)
 
             return floats;
         }
+
+        // This REQUIRES you to use Marshal.FreeHGlobal() on the returned pointer!
+        public static IntPtr StringToUtf8(String managedString)
+        {
+            int len = Encoding.UTF8.GetByteCount(managedString);
+
+            byte[] buffer = new byte[len + 1];
+
+            Encoding.UTF8.GetBytes(managedString, 0, managedString.Length, buffer, 0);
+
+            IntPtr nativeUtf8 = Marshal.AllocHGlobal(buffer.Length);
+
+            Marshal.Copy(buffer, 0, nativeUtf8, buffer.Length);
+
+            return nativeUtf8;
+        }
+
+        // this version will also return the length of the utf8 string
+        // This REQUIRES you to use Marshal.FreeHGlobal() on the returned pointer!
+        public static IntPtr StringToUtf8(String managedString, out int utf8Length)
+        {
+            utf8Length = Encoding.UTF8.GetByteCount(managedString);
+
+            byte[] buffer = new byte[utf8Length + 1];
+
+            Encoding.UTF8.GetBytes(managedString, 0, managedString.Length, buffer, 0);
+
+            IntPtr nativeUtf8 = Marshal.AllocHGlobal(buffer.Length);
+
+            Marshal.Copy(buffer, 0, nativeUtf8, buffer.Length);
+
+            return nativeUtf8;
+        }
+
+        // Length is optional, but recommended
+        // This is all potentially dangerous
+        public static string Utf8ToString(IntPtr nativeUtf8, uint? length = null)
+        {
+            if (nativeUtf8 == IntPtr.Zero)
+                return String.Empty;
+
+            uint len = 0;
+
+            if (length.HasValue)
+            {
+                len = length.Value;
+            }
+            else
+            {
+                // try to find the terminator
+                while (Marshal.ReadByte(nativeUtf8, (int)len) != 0)
+                {
+                    ++len;
+                }
+            }
+
+            byte[] buffer = new byte[len];
+
+            Marshal.Copy(nativeUtf8, buffer, 0, buffer.Length);
+
+            return Encoding.UTF8.GetString(buffer);
+        }
     }
 
     //

Packages/jp.keijiro.klak.ndi/Runtime/Interop/Recv.cs

@@ -60,12 +60,22 @@ public FrameType Capture
       (out VideoFrame video, out AudioFrame audio, IntPtr metadata, uint timeout)
       => _Capture(this, out video, out audio, metadata, timeout);
 
+    public FrameType Capture
+      (out VideoFrame video, out AudioFrame audio, out MetadataFrame metadata, uint timeout)
+      => _Capture(this, out video, out audio, out metadata, timeout);
+
     public void FreeVideoFrame(in VideoFrame frame)
-      => _FreeVideo(this, frame); 
-        
+      => _FreeVideo(this, frame);
+
     public void FreeAudioFrame(in AudioFrame frame)
       => _FreeAudio(this, frame);
 
+    public void FreeMetadataFrame(in MetadataFrame frame)
+      => _FreeMetadata(this, frame);
+
+    public void SendMetadata(in MetadataFrame frame)
+      => _SendMetadata(this, frame);
+
     public bool SetTally(in Tally tally)
       => _SetTally(this, tally);
 
@@ -90,12 +100,23 @@ static extern FrameType _Capture(Recv recv,
     static extern FrameType _Capture(Recv recv,
       out VideoFrame video, out AudioFrame audio, IntPtr metadata, uint timeout);
 
+    [DllImport(Config.DllName, EntryPoint = "NDIlib_recv_capture_v2")]
+    static extern FrameType _Capture(Recv recv,
+      out VideoFrame video, out AudioFrame audio, out MetadataFrame metadata, uint timeout);
+
     [DllImport(Config.DllName, EntryPoint = "NDIlib_recv_free_video_v2")]
     static extern void _FreeVideo(Recv recv, in VideoFrame data);
-        
+
     [DllImport(Config.DllName, EntryPoint = "NDIlib_recv_free_audio_v2")]
     static extern void _FreeAudio(Recv recv, in AudioFrame data);
 
+    [DllImport(Config.DllName, EntryPoint = "NDIlib_recv_free_metadata")]
+    static extern void _FreeMetadata(Recv recv, in MetadataFrame data);
+
+    [DllImport(Config.DllName, EntryPoint = "NDIlib_recv_send_metadata")]
+    [return: MarshalAsAttribute(UnmanagedType.U1)]
+    static extern bool _SendMetadata(Recv recv, in MetadataFrame data);
+
     [DllImport(Config.DllName, EntryPoint = "NDIlib_recv_set_tally")]
     [return: MarshalAs(UnmanagedType.U1)]
     static extern bool _SetTally(Recv recv, in Tally tally);