🕑 Support playout delay extension (#134)

Adds support for the experimental playout-delay RTP extension.

See: https://webrtc.googlesource.com/src/+/refs/heads/main/docs/native-code/rtp-hdrext/playout-delay

meson.build

@@ -69,6 +69,10 @@ if systemd_dep.found()
   add_project_arguments('-DSYSTEMD_WATCHDOG_SUPPORT', language: 'cpp') 
 endif
 
+if get_option('janus_playout_delay_support')
+    add_project_arguments('-DJANUS_PLAYOUT_DELAY_SUPPORT', language: 'cpp')
+endif
+
 deps = [
     dependency('glib-2.0'),
     dependency('libsrtp2'),

meson_options.txt

@@ -1 +1,2 @@
 option('systemd_watchdog_support', type : 'feature', value : 'auto') # http://0pointer.de/blog/projects/watchdog.html
+option('janus_playout_delay_support', type : 'boolean', value: false)

src/Configuration.cpp

@@ -19,6 +19,8 @@
 #include <unistd.h>
 #include <wordexp.h>
 
+using namespace std::chrono_literals;
+
 #pragma region Public methods
 void Configuration::Load()
 {
@@ -79,7 +81,7 @@ void Configuration::Load()
     // FTL_ORCHESTRATOR_PORT -> OrchestratorPort
     if (char* varVal = std::getenv("FTL_ORCHESTRATOR_PORT"))
     {
-        orchestratorPort = std::stoi(varVal);
+        orchestratorPort = std::stoul(varVal);
     }
 
     // FTL_ORCHESTRATOR_PSK -> OrchestratorPsk
@@ -120,25 +122,49 @@ void Configuration::Load()
     // FTL_SERVICE_METADATAREPORTINTERVALMS -> ServiceConnectionMetadataReportInterval
     if (char* varVal = std::getenv("FTL_SERVICE_METADATAREPORTINTERVALMS"))
     {
-        serviceConnectionMetadataReportInterval = std::chrono::milliseconds(std::stoi(varVal));
+        serviceConnectionMetadataReportInterval = std::chrono::milliseconds(std::stoul(varVal));
     }
 
     // FTL_MAX_ALLOWED_BITS_PER_SECOND -> MaxAllowedBitsPerSecond
     if (char* varVal = std::getenv("FTL_MAX_ALLOWED_BITS_PER_SECOND"))
     {
-        maxAllowedBitsPerSecond = std::stoi(varVal);
+        maxAllowedBitsPerSecond = std::stoul(varVal);
     }
 
     // FTL_ROLLING_SIZE_AVG_MS -> RollingSizeAvgMs
     if (char* varVal = std::getenv("FTL_ROLLING_SIZE_AVG_MS"))
     {
-        rollingSizeAvgMs = std::stoi(varVal);
+        rollingSizeAvgMs = std::stoul(varVal);
     }
 
     // FTL_NACK_LOST_PACKETS -> IsNackLostPacketsEnabled
     if (char* varVal = std::getenv("FTL_NACK_LOST_PACKETS"))
     {
-        nackLostPackets = std::stoi(varVal);
+        nackLostPackets = std::stoul(varVal);
+    }
+
+    // FTL_PLAYOUT_DELAY_MIN_MS and FTL_PLAYOUT_DELAY_MAX_MS -> PlayoutDelay
+    {
+        char* minVal = std::getenv("FTL_PLAYOUT_DELAY_MIN_MS");
+        char* maxVal = std::getenv("FTL_PLAYOUT_DELAY_MAX_MS");
+
+        if (minVal || maxVal)
+        {
+            if (!PLAYOUT_DELAY_SUPPORT)
+            {
+                spdlog::warn("Ignoring playout delay configuration, option janus_playout_delay_support is not enabled");
+            }
+            else if (minVal && maxVal)
+            {
+                auto min = std::chrono::milliseconds(std::stoul(minVal));
+                auto max = std::chrono::milliseconds(std::stoul(maxVal));
+                playoutDelay = PlayoutDelay(min, max);
+            }
+            else
+            {
+                throw InvalidConfigurationException("Both min and max playout delay values must be set together");
+            }
+        }
     }
 
     // FTL_SERVICE_DUMMY_HMAC_KEY -> DummyHmacKey
@@ -172,13 +198,13 @@ void Configuration::Load()
     // FTL_SERVICE_GLIMESH_PORT -> GlimeshServicePort
     if (char* varVal = std::getenv("FTL_SERVICE_GLIMESH_PORT"))
     {
-        glimeshServicePort = std::stoi(varVal);
+        glimeshServicePort = std::stoul(varVal);
     }
 
     // FTL_SERVICE_GLIMESH_HTTPS -> GlimeshServiceUseHttps
     if (char* varVal = std::getenv("FTL_SERVICE_GLIMESH_HTTPS"))
     {
-        glimeshServiceUseHttps = std::stoi(varVal);
+        glimeshServiceUseHttps = std::stoul(varVal);
     }
 
     // FTL_SERVICE_GLIMESH_CLIENTID -> GlimeshServiceClientId
@@ -202,13 +228,13 @@ void Configuration::Load()
     // FTL_SERVICE_REST_PORT -> RestServicePort
     if (char* varVal = std::getenv("FTL_SERVICE_REST_PORT"))
     {
-        restServicePort = std::stoi(varVal);
+        restServicePort = std::stoul(varVal);
     }
 
     // FTL_SERVICE_REST_HTTPS -> RestServiceUseHttps
     if (char* varVal = std::getenv("FTL_SERVICE_REST_HTTPS"))
     {
-        restServiceUseHttps = std::stoi(varVal);
+        restServiceUseHttps = std::stoul(varVal);
     }
 
     // FTL_SERVICE_REST_PATH_BASE -> RestServicePathBase
@@ -223,6 +249,34 @@ void Configuration::Load()
         restServiceAuthToken = std::string(varVal);
     }
 }
+
+Configuration::PlayoutDelay::PlayoutDelay(std::chrono::milliseconds min_ms, std::chrono::milliseconds max_ms)
+{
+    if (min_ms < 0ms)
+    {
+        throw InvalidConfigurationException("Playout delay min must be greater than or equal to 0");
+    }
+    if (min_ms > 40950ms)
+    {
+        throw InvalidConfigurationException("FTL_PLAYOUT_DELAY_MIN_MS must be less than or equal to 40950ms");
+    }
+    if (max_ms < 0ms)
+    {
+        throw InvalidConfigurationException("Playout delay max must be greater than or equal to 0");
+    }
+    if (max_ms > 40950ms)
+    {
+        throw InvalidConfigurationException("FTL_PLAYOUT_DELAY_MAX_MS must be less than or equal to 40950ms");
+    }
+    if (min_ms > max_ms)
+    {
+        throw InvalidConfigurationException("FTL_PLAYOUT_DELAY_MIN_MS cannot be greater than FTL_PLAYOUT_DELAY_MAX_MS");
+    }
+
+    // Convert validated values to units of 10ms as used in the playout-delay header extension
+    min = min_ms.count() / 10;
+    max = max_ms.count() / 10;
+}
 #pragma endregion
 
 #pragma region Configuration values
@@ -291,6 +345,11 @@ bool Configuration::IsNackLostPacketsEnabled()
     return nackLostPackets;
 }
 
+std::optional<Configuration::PlayoutDelay> Configuration::GetPlayoutDelay()
+{
+    return playoutDelay;
+}
+
 std::string Configuration::GetGlimeshServiceHostname()
 {
     return glimeshServiceHostname;
@@ -340,6 +399,7 @@ std::string Configuration::GetRestServiceAuthToken()
 {
     return restServiceAuthToken;
 }
+
 #pragma endregion
 
 #pragma region Private methods

src/Configuration.h

@@ -12,6 +12,7 @@
 
 #include <chrono>
 #include <cstdint>
+#include <optional>
 #include <string>
 #include <vector>
 
@@ -33,6 +34,42 @@ enum class ServiceConnectionKind
 class Configuration
 {
 public:
+    /**
+     * @brief Range of expected delay between server capturing a frame and clients receiving it.
+     * 
+     * Note min/max delay are in units of 10ms as specified by the playout-delay specification, but
+     * the constructor takes the chrono::duration type for clarity and convenience.
+     * 
+     * Sent to clients via an experimental RTP extension only implemented for Chrome. Can be used to
+     * suggest a bounded range client should delay before rendering a frame. In theory the client
+     * should determine an appropriate delay to account for network jitter and rendering time.
+     * 
+     * However, we have seen Chrome be wrong when choosing a delay, and there are other use cases
+     * where bounding the minimum or maximum delay can be useful. See the RFC for more details.
+     * https://webrtc.googlesource.com/src/+/refs/heads/main/docs/native-code/rtp-hdrext/playout-delay
+     * 
+     * Reasonable values range from 0 to 10,000 milliseconds (rounded to a granularity of 10ms). The
+     * ideal value depends on expected network delay and jitter clients will experience. Generally
+     * a minimum of 100ms-400ms and a maximum of a few seconds is a good starting range. 
+     */
+    struct PlayoutDelay
+    {
+public:
+        PlayoutDelay(std::chrono::milliseconds min_ms, std::chrono::milliseconds max_ms);
+
+        /* Public methods */
+        uint16_t MinDelay() {
+            return min;
+        }
+        uint16_t MaxDelay() {
+            return max;
+        }
+
+private:
+        uint16_t min;
+        uint16_t max;
+    };
+
     /* Public methods */
     void Load();
 
@@ -48,6 +85,7 @@ class Configuration
     uint32_t GetMaxAllowedBitsPerSecond();
     uint32_t GetRollingSizeAvgMs();
     bool IsNackLostPacketsEnabled();
+    std::optional<PlayoutDelay> GetPlayoutDelay();
 
     // Dummy Service Connection Values
     std::vector<std::byte> GetDummyHmacKey();
@@ -68,6 +106,16 @@ class Configuration
     std::string GetRestServiceAuthToken();
 
 private:
+    /* Constants */
+    // Playout delay configuration can only used if your Janus version supports the playout-delay
+    // RTP extension, hence the compiler flag. We use this constant to print a warning to the user
+    // if they set a delay configuration but it is not being used.
+    #if defined(JANUS_PLAYOUT_DELAY_SUPPORT)
+    static constexpr bool PLAYOUT_DELAY_SUPPORT = true;
+    #else
+    static constexpr bool PLAYOUT_DELAY_SUPPORT = false;
+    #endif
+
     /* Backing stores */
     std::string myHostname;
     NodeKind nodeKind = NodeKind::Standalone;
@@ -80,6 +128,7 @@ class Configuration
     uint32_t maxAllowedBitsPerSecond = 0;
     uint32_t rollingSizeAvgMs = 2000;
     bool nackLostPackets = false;
+    std::optional<PlayoutDelay> playoutDelay;
 
     // Dummy Service Connection Backing Stores
     // "aBcDeFgHiJkLmNoPqRsTuVwXyZ123456"
@@ -113,4 +162,14 @@ class Configuration
      * @brief Takes a hex string of format "010203FF" and converts it to an array of bytes.
      */
     std::vector<std::byte> hexStringToByteArray(std::string hexString);
-};
+};
+
+/**
+ * @brief Exception describing invalid configuration values
+ */
+struct InvalidConfigurationException : std::runtime_error
+{
+    InvalidConfigurationException(const char *message) throw() : std::runtime_error(message)
+    {
+    }
+};

src/FtlClient.cpp

@@ -110,12 +110,12 @@ void FtlClient::SetOnClosed(std::function<void()> onClosed)
     this->onClosed = onClosed;
 }
 
-void FtlClient::RelayPacket(const std::vector<std::byte>& packet)
+void FtlClient::RelayPacket(const RtpPacket& packet)
 {
     if (mediaSocketHandle != 0)
     {
-        size_t writeResult = write(mediaSocketHandle, packet.data(), packet.size());
-        if (writeResult != packet.size())
+        size_t writeResult = write(mediaSocketHandle, packet.Bytes.data(), packet.Bytes.size());
+        if (writeResult != packet.Bytes.size())
         {
             // TODO: Handle writeResult
         }

src/FtlClient.h

@@ -7,6 +7,7 @@
 
 #pragma once
 
+#include "Rtp/RtpPacket.h"
 #include "Utilities/FtlTypes.h"
 #include "Utilities/Result.h"
 
@@ -79,7 +80,7 @@ class FtlClient
     /**
      * @brief Relays a packet from an incoming FtlStream
      */
-    void RelayPacket(const std::vector<std::byte>& packet);
+    void RelayPacket(const RtpPacket& packet);
 
 private:
     /* Private structs */

src/FtlMediaConnection.cpp

@@ -29,7 +29,7 @@ FtlMediaConnection::FtlMediaConnection(
     const ftl_channel_id_t channelId,
     const ftl_stream_id_t streamId,
     const ClosedCallback onClosed,
-    const RtpPacketCallback onRtpPacketBytes,
+    const RtpPacketCallback onRtpPacket,
     const uint32_t rollingSizeAvgMs,
     const bool nackLostPackets)
 :
@@ -38,7 +38,7 @@ FtlMediaConnection::FtlMediaConnection(
     channelId(channelId),
     streamId(streamId),
     onClosed(onClosed),
-    onRtpPacketBytes(onRtpPacketBytes),
+    onRtpPacket(onRtpPacket),
     rollingSizeAvgMs(rollingSizeAvgMs),
     nackLostPackets(nackLostPackets),
     thread(std::jthread(std::bind(&FtlMediaConnection::threadBody, this, std::placeholders::_1)))
@@ -577,9 +577,9 @@ void FtlMediaConnection::processAudioVideoRtpPacket(const RtpPacket& rtpPacket,
 {
     processRtpPacketKeyframe(rtpPacket, dataLock);
 
-    if (onRtpPacketBytes)
+    if (onRtpPacket)
     {
-        onRtpPacketBytes(rtpPacket.Bytes);
+        onRtpPacket(rtpPacket);
     }
 }
 

src/FtlMediaConnection.h

@@ -37,7 +37,7 @@ class FtlMediaConnection
 public:
     /* Public types */
     using ClosedCallback = std::function<void(FtlMediaConnection&)>;
-    using RtpPacketCallback = std::function<void(const std::vector<std::byte>&)>;
+    using RtpPacketCallback = std::function<void(const RtpPacket&)>;
 
     /* Constructor/Destructor */
     FtlMediaConnection(
@@ -91,7 +91,7 @@ class FtlMediaConnection
     const ftl_channel_id_t channelId;
     const ftl_stream_id_t streamId;
     const ClosedCallback onClosed;
-    const RtpPacketCallback onRtpPacketBytes;
+    const RtpPacketCallback onRtpPacket;
     const uint32_t rollingSizeAvgMs;
     const bool nackLostPackets;
     // Stream data

src/FtlServer.cpp

@@ -579,7 +579,7 @@ void FtlServer::eventStreamIdAssigned(std::shared_ptr<FtlServerStreamIdAssignedE
                 std::move(mediaTransport),
                 mediaPort,
                 event->Metadata,
-                [rtpPacketSink](const std::vector<std::byte> packet)
+                [rtpPacketSink](const RtpPacket& packet)
                 {
                     rtpPacketSink->SendRtpPacket(packet);
                 });

src/JanusFtl.cpp

@@ -46,13 +46,15 @@ JanusFtl::JanusFtl(
 #else
     spdlog::set_level(spdlog::level::info);
 #endif
+    spdlog::flush_on(spdlog::level::err);
 
     configuration = std::make_unique<Configuration>();
     configuration->Load();
     maxAllowedBitsPerSecond = configuration->GetMaxAllowedBitsPerSecond();
     rollingSizeAvgMs = configuration->GetRollingSizeAvgMs();
     metadataReportInterval = configuration->GetServiceConnectionMetadataReportInterval();
     watchdog = std::make_unique<Watchdog>(configuration->GetServiceConnectionMetadataReportInterval());
+    playoutDelay = configuration->GetPlayoutDelay();
 
     initVideoDecoders();
 
@@ -96,7 +98,7 @@ JanusFtl::~JanusFtl()
 void JanusFtl::CreateSession(janus_plugin_session* handle, int* error)
 {
     std::unique_lock lock(streamDataMutex);
-    auto session = std::make_unique<JanusSession>(handle, janusCore);
+    auto session = std::make_unique<JanusSession>(handle, janusCore, playoutDelay);
     handle->plugin_handle = session.get();
     sessions[handle] = ActiveSession
     {
@@ -881,7 +883,8 @@ std::string JanusFtl::generateSdpOffer(const ActiveSession& session, const Janus
             "a=rtcp-fb:" << videoPayloadType << " nack pli\r\n" <<        // Send us picture-loss-indicators
             // "a=rtcp-fb:96 nack goog-remb\r\n" <<  // Send some congestion indicator thing
             "a=sendonly\r\n" <<
-            "a=extmap:1 urn:ietf:params:rtp-hdrext:sdes:mid\r\n";
+            "a=extmap:1 urn:ietf:params:rtp-hdrext:sdes:mid\r\n" <<
+            "a=extmap:6 http://www.webrtc.org/experiments/rtp-hdrext/playout-delay\r\n";
     }
     return offerStream.str();
 }