From 9b345fca69e2ba6d78c946cca7a2c9b4d480d7fa Mon Sep 17 00:00:00 2001 From: Ilias Pavlidakis Date: Fri, 13 Mar 2026 13:43:23 +0200 Subject: [PATCH 1/3] [Enhancement]Forward active encoder info when SEA has one unpaused layer --- media/engine/simulcast_encoder_adapter.cc | 32 ++++ .../simulcast_encoder_adapter_unittest.cc | 139 ++++++++++++++++++ 2 files changed, 171 insertions(+) diff --git a/media/engine/simulcast_encoder_adapter.cc b/media/engine/simulcast_encoder_adapter.cc index 73040a485f..cb9892a9d2 100644 --- a/media/engine/simulcast_encoder_adapter.cc +++ b/media/engine/simulcast_encoder_adapter.cc @@ -951,6 +951,12 @@ VideoEncoder::EncoderInfo SimulcastEncoderAdapter::GetEncoderInfo() const { encoder_info.scaling_settings = VideoEncoder::ScalingSettings::kOff; std::vector encoder_names; + // SEA can keep multiple stream contexts alive even when runtime bitrate + // allocation has paused all but one spatial layer. Track that state + // explicitly so we can preserve simulcast-specific aggregation while still + // forwarding the active encoder's single-layer adaptation hints. + size_t active_stream_count = 0; + std::optional active_stream_info; for (size_t i = 0; i < stream_contexts_.size(); ++i) { VideoEncoder::EncoderInfo encoder_impl_info = @@ -958,6 +964,11 @@ VideoEncoder::EncoderInfo SimulcastEncoderAdapter::GetEncoderInfo() const { // Encoder name indicates names of all active sub-encoders. if (!stream_contexts_[i].is_paused()) { + // If exactly one layer stays unpaused after SetRates(), this is the + // encoder whose runtime adaptation fields should be exposed to the rest + // of WebRTC. + ++active_stream_count; + active_stream_info = encoder_impl_info; encoder_names.push_back(encoder_impl_info.implementation_name); } if (i == 0) { @@ -1011,6 +1022,27 @@ VideoEncoder::EncoderInfo SimulcastEncoderAdapter::GetEncoderInfo() const { encoder_info.implementation_name += implementation_name_builder.Release(); } + if (active_stream_count == 1) { + RTC_DCHECK(active_stream_info.has_value()); + // Keep the aggregated SEA view for simulcast-specific fields such as + // implementation_name, fps_allocation and alignment, but make the adapter + // behave like single-layer publishing for the runtime-sensitive fields + // consumed by adaptation and frame handling. + encoder_info.scaling_settings = active_stream_info->scaling_settings; + encoder_info.supports_native_handle = + active_stream_info->supports_native_handle; + encoder_info.has_trusted_rate_controller = + active_stream_info->has_trusted_rate_controller; + encoder_info.is_hardware_accelerated = + active_stream_info->is_hardware_accelerated; + encoder_info.is_qp_trusted = active_stream_info->is_qp_trusted; + encoder_info.resolution_bitrate_limits = + active_stream_info->resolution_bitrate_limits; + encoder_info.min_qp = active_stream_info->min_qp; + encoder_info.preferred_pixel_formats = + active_stream_info->preferred_pixel_formats; + } + OverrideFromFieldTrial(&encoder_info); return encoder_info; diff --git a/media/engine/simulcast_encoder_adapter_unittest.cc b/media/engine/simulcast_encoder_adapter_unittest.cc index a332003054..4101d41216 100644 --- a/media/engine/simulcast_encoder_adapter_unittest.cc +++ b/media/engine/simulcast_encoder_adapter_unittest.cc @@ -286,6 +286,7 @@ class MockVideoEncoder : public VideoEncoder { info.supports_simulcast = supports_simulcast_; info.is_qp_trusted = is_qp_trusted_; info.resolution_bitrate_limits = resolution_bitrate_limits; + info.min_qp = min_qp_; return info; } @@ -365,6 +366,8 @@ class MockVideoEncoder : public VideoEncoder { resolution_bitrate_limits = limits; } + void set_min_qp(std::optional min_qp) { min_qp_ = min_qp; } + bool supports_simulcast() const { return supports_simulcast_; } SdpVideoFormat video_format() const { return video_format_; } @@ -384,6 +387,7 @@ class MockVideoEncoder : public VideoEncoder { FramerateFractions fps_allocation_; bool supports_simulcast_ = false; std::optional is_qp_trusted_; + std::optional min_qp_; SdpVideoFormat video_format_; std::vector resolution_bitrate_limits; @@ -1692,6 +1696,141 @@ TEST_F(TestSimulcastEncoderAdapterFake, ReportsFpsAllocation) { ::testing::ElementsAreArray(expected_fps_allocation)); } +TEST_F(TestSimulcastEncoderAdapterFake, + ForwardsRuntimeSensitiveEncoderInfoForSingleUnpausedLayer) { + SimulcastTestFixtureImpl::DefaultSettings( + &codec_, static_cast(kTestTemporalLayerProfile), + kVideoCodecVP8); + codec_.numberOfSimulcastStreams = 3; + EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); + adapter_->RegisterEncodeCompleteCallback(this); + ASSERT_EQ(3u, helper_->factory()->encoders().size()); + + auto* low_encoder = helper_->factory()->encoders()[0]; + auto* mid_encoder = helper_->factory()->encoders()[1]; + auto* high_encoder = helper_->factory()->encoders()[2]; + + low_encoder->set_scaling_settings(VideoEncoder::ScalingSettings(10, 20, 111)); + low_encoder->set_supports_native_handle(false); + low_encoder->set_is_qp_trusted(true); + low_encoder->set_resolution_bitrate_limits( + {VideoEncoder::ResolutionBitrateLimits(111, 1111, 2222, 3333)}); + low_encoder->set_min_qp(10); + low_encoder->set_fps_allocation( + FramerateFractions{EncoderInfo::kMaxFramerateFraction / 2}); + + mid_encoder->set_scaling_settings(VideoEncoder::ScalingSettings(30, 40, 222)); + mid_encoder->set_supports_native_handle(true); + mid_encoder->set_is_qp_trusted(false); + mid_encoder->set_resolution_bitrate_limits( + {VideoEncoder::ResolutionBitrateLimits(222, 4444, 5555, 6666)}); + mid_encoder->set_min_qp(20); + mid_encoder->set_fps_allocation( + FramerateFractions{EncoderInfo::kMaxFramerateFraction / 3, + EncoderInfo::kMaxFramerateFraction}); + + high_encoder->set_scaling_settings( + VideoEncoder::ScalingSettings(50, 60, 333)); + high_encoder->set_supports_native_handle(false); + high_encoder->set_is_qp_trusted(true); + high_encoder->set_resolution_bitrate_limits( + {VideoEncoder::ResolutionBitrateLimits(333, 7777, 8888, 9999)}); + high_encoder->set_min_qp(30); + high_encoder->set_fps_allocation( + FramerateFractions{EncoderInfo::kMaxFramerateFraction}); + + // Only keep the middle spatial layer active. SEA still has three stream + // contexts, so this exercises the runtime state that used to incorrectly + // report aggregated simulcast encoder info with scaling disabled. + VideoBitrateAllocation allocation; + ASSERT_TRUE(allocation.SetBitrate(1, 0, 500000)); + adapter_->SetRates(VideoEncoder::RateControlParameters(allocation, 30.0)); + + const auto info = adapter_->GetEncoderInfo(); + // Runtime-sensitive fields should come from the only unpaused encoder. + ASSERT_TRUE(info.scaling_settings.thresholds.has_value()); + EXPECT_EQ(30, info.scaling_settings.thresholds->low); + EXPECT_EQ(40, info.scaling_settings.thresholds->high); + EXPECT_EQ(222, info.scaling_settings.min_pixels_per_frame); + EXPECT_TRUE(info.supports_native_handle); + EXPECT_EQ(std::optional(false), info.is_qp_trusted); + EXPECT_EQ(std::optional(20), info.min_qp); + EXPECT_EQ(info.resolution_bitrate_limits, + std::vector( + {VideoEncoder::ResolutionBitrateLimits(222, 4444, 5555, 6666)})); + // Simulcast-specific fields must remain in SEA's aggregated spatial-slot + // layout even when runtime-sensitive fields are forwarded from one encoder. + EXPECT_THAT(info.fps_allocation[0], ::testing::IsEmpty()); + EXPECT_THAT(info.fps_allocation[1], + ::testing::ElementsAre(EncoderInfo::kMaxFramerateFraction / 3, + EncoderInfo::kMaxFramerateFraction)); + EXPECT_THAT(info.fps_allocation[2], + ::testing::ElementsAre(EncoderInfo::kMaxFramerateFraction)); +} + +TEST_F(TestSimulcastEncoderAdapterFake, + RestoresAggregatedEncoderInfoWhenMultipleLayersUnpause) { + SimulcastTestFixtureImpl::DefaultSettings( + &codec_, static_cast(kTestTemporalLayerProfile), + kVideoCodecVP8); + codec_.numberOfSimulcastStreams = 3; + EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); + adapter_->RegisterEncodeCompleteCallback(this); + ASSERT_EQ(3u, helper_->factory()->encoders().size()); + + auto* low_encoder = helper_->factory()->encoders()[0]; + auto* mid_encoder = helper_->factory()->encoders()[1]; + auto* high_encoder = helper_->factory()->encoders()[2]; + + low_encoder->set_scaling_settings(VideoEncoder::ScalingSettings(10, 20, 111)); + low_encoder->set_supports_native_handle(false); + low_encoder->set_fps_allocation( + FramerateFractions{EncoderInfo::kMaxFramerateFraction / 2}); + + mid_encoder->set_scaling_settings(VideoEncoder::ScalingSettings(30, 40, 222)); + mid_encoder->set_supports_native_handle(true); + mid_encoder->set_fps_allocation( + FramerateFractions{EncoderInfo::kMaxFramerateFraction / 3, + EncoderInfo::kMaxFramerateFraction}); + + high_encoder->set_scaling_settings( + VideoEncoder::ScalingSettings(50, 60, 333)); + high_encoder->set_supports_native_handle(false); + high_encoder->set_fps_allocation( + FramerateFractions{EncoderInfo::kMaxFramerateFraction}); + + // First collapse to a single active spatial layer and verify the forwarded + // encoder info. + VideoBitrateAllocation one_layer_allocation; + ASSERT_TRUE(one_layer_allocation.SetBitrate(1, 0, 500000)); + adapter_->SetRates( + VideoEncoder::RateControlParameters(one_layer_allocation, 30.0)); + + auto info = adapter_->GetEncoderInfo(); + ASSERT_TRUE(info.scaling_settings.thresholds.has_value()); + EXPECT_EQ(30, info.scaling_settings.thresholds->low); + EXPECT_EQ(40, info.scaling_settings.thresholds->high); + EXPECT_TRUE(info.supports_native_handle); + + // Then enable another layer. SEA should immediately return to its normal + // aggregated simulcast view without requiring a re-init. + VideoBitrateAllocation two_layer_allocation; + ASSERT_TRUE(two_layer_allocation.SetBitrate(1, 0, 500000)); + ASSERT_TRUE(two_layer_allocation.SetBitrate(2, 0, 700000)); + adapter_->SetRates( + VideoEncoder::RateControlParameters(two_layer_allocation, 30.0)); + + info = adapter_->GetEncoderInfo(); + EXPECT_FALSE(info.scaling_settings.thresholds.has_value()); + EXPECT_TRUE(info.supports_native_handle); + EXPECT_THAT(info.fps_allocation[0], ::testing::IsEmpty()); + EXPECT_THAT(info.fps_allocation[1], + ::testing::ElementsAre(EncoderInfo::kMaxFramerateFraction / 3, + EncoderInfo::kMaxFramerateFraction)); + EXPECT_THAT(info.fps_allocation[2], + ::testing::ElementsAre(EncoderInfo::kMaxFramerateFraction)); +} + TEST_F(TestSimulcastEncoderAdapterFake, SetRateDistributesBandwithAllocation) { SimulcastTestFixtureImpl::DefaultSettings( &codec_, static_cast(kTestTemporalLayerProfile), From 5008872d4c032c922ac0a64ca58029e7947cdd51 Mon Sep 17 00:00:00 2001 From: Stream Bot Date: Mon, 30 Mar 2026 17:02:18 +0300 Subject: [PATCH 2/3] [Enhancement] Add GetMuted method to FakeAudioSendStream and improve encoder stall handling in RTCVideoEncoderH264 and VideoSendStreamImpl - Implemented GetMuted method in FakeAudioSendStream to align with AudioSendStream interface. - Introduced encoder stall detection and recovery logic in RTCVideoEncoderH264, including timestamps for successful encodes and forced resets. - Enhanced VideoSendStreamImpl to track encoder timeouts and recovery events for better telemetry and operational insights. # Conflicts: # media/engine/fake_webrtc_call.h --- media/engine/fake_webrtc_call.cc | 4 ++ media/engine/fake_webrtc_call.h | 3 +- .../video_codec/RTCVideoEncoderH264.mm | 61 +++++++++++++++++++ video/video_send_stream_impl.cc | 32 +++++++++- video/video_send_stream_impl.h | 6 ++ 5 files changed, 103 insertions(+), 3 deletions(-) diff --git a/media/engine/fake_webrtc_call.cc b/media/engine/fake_webrtc_call.cc index fb8d985a14..4b8020acf7 100644 --- a/media/engine/fake_webrtc_call.cc +++ b/media/engine/fake_webrtc_call.cc @@ -99,6 +99,10 @@ void FakeAudioSendStream::SetMuted(bool muted) { muted_ = muted; } +bool FakeAudioSendStream::GetMuted() { + return muted_; +} + AudioSendStream::Stats FakeAudioSendStream::GetStats() const { return stats_; } diff --git a/media/engine/fake_webrtc_call.h b/media/engine/fake_webrtc_call.h index f130977fb8..a6930ee7b5 100644 --- a/media/engine/fake_webrtc_call.h +++ b/media/engine/fake_webrtc_call.h @@ -100,7 +100,8 @@ class FakeAudioSendStream final : public AudioSendStream { int payload_frequency, int event, int duration_ms) override; - bool GetMuted() override { return muted_; } + // Keep fake stream API aligned with AudioSendStream interface changes. + bool GetMuted() override; void SetMuted(bool muted) override; AudioSendStream::Stats GetStats() const override; AudioSendStream::Stats GetStats(bool has_remote_tracks) const override; diff --git a/sdk/objc/components/video_codec/RTCVideoEncoderH264.mm b/sdk/objc/components/video_codec/RTCVideoEncoderH264.mm index d2ec2dbe8f..dd15be001b 100644 --- a/sdk/objc/components/video_codec/RTCVideoEncoderH264.mm +++ b/sdk/objc/components/video_codec/RTCVideoEncoderH264.mm @@ -61,6 +61,11 @@ - (void)frameWasEncoded : (OSStatus)status flags const int kLowH264QpThreshold = 28; const int kHighH264QpThreshold = 39; const int kBitsPerByte = 8; +// If no encoded frame is observed for this long while bitrate is non-zero, +// treat the encoder as stalled and force a session reset. +const int64_t kEncoderStallResetThresholdMs = 2500; +// Minimum gap between forced resets to avoid rapid reset loops. +const int64_t kEncoderStallMinResetIntervalMs = 1000; const OSType kNV12PixelFormat = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange; @@ -384,6 +389,12 @@ @implementation RTC_OBJC_TYPE (RTCVideoEncoderH264) { std::vector _frameScaleBuffer; CMTime _previousPresentationTimeStamp; + // Timestamp (ms) of the last frame successfully produced by VideoToolbox. + int64_t _lastSuccessfulEncodeMs; + // Timestamp (ms) of the last hard reset we triggered for stall recovery. + int64_t _lastHardResetMs; + // True after a forced reset until we observe the first successful frame. + BOOL _isInForcedRecovery; } // .5 is set as a mininum to prevent overcompensating for large temporary @@ -402,6 +413,12 @@ - (instancetype)initWithCodecInfo: _profile_level_id = webrtc::ParseSdpForH264ProfileLevelId([codecInfo nativeSdpVideoFormat].parameters); _previousPresentationTimeStamp = kCMTimeZero; + // Zero means "no successful encode yet" for this encoder instance. + _lastSuccessfulEncodeMs = 0; + // Zero means "no forced reset has happened yet". + _lastHardResetMs = 0; + // Start in normal mode; recovery mode is enabled only after forced reset. + _isInForcedRecovery = NO; RTC_DCHECK(_profile_level_id); RTC_LOG(LS_INFO) << "Using profile " << CFStringToString(ExtractProfile( @@ -445,6 +462,12 @@ - (NSInteger)startEncodeWithSettings: _targetFrameRate = MIN(settings.maxFramerate, _maxAllowedFrameRate); _encoderBitrateBps = 0; _encoderFrameRate = 0; + // Reset stall tracking when a fresh encoding session starts. + _lastSuccessfulEncodeMs = 0; + // Reset last-reset timestamp for the new session lifecycle. + _lastHardResetMs = 0; + // New session starts outside recovery mode. + _isInForcedRecovery = NO; if (settings.maxFramerate > _maxAllowedFrameRate && _maxAllowedFrameRate > 0) { RTC_LOG(LS_WARNING) << "Initial encoder frame rate setting " << settings.maxFramerate << " is larger than the " @@ -472,6 +495,31 @@ - (NSInteger)encode:(RTC_OBJC_TYPE(RTCVideoFrame) *)frame } _previousPresentationTimeStamp = presentationTimeStamp; +#if defined(WEBRTC_IOS) + // Evaluate stall heuristics only on iOS where this recovery path is required. + const int64_t nowMs = rtc::TimeMillis(); + // Trigger forced reset only when: + // 1) encoder should be producing data (_targetBitrateBps > 0), + // 2) session exists, + // 3) we've seen at least one successful encode before, + // 4) stall duration exceeds threshold, + // 5) minimum interval since previous reset has elapsed. + if (_targetBitrateBps > 0 && _compressionSession && _lastSuccessfulEncodeMs > 0 && + nowMs - _lastSuccessfulEncodeMs >= kEncoderStallResetThresholdMs && + nowMs - _lastHardResetMs >= kEncoderStallMinResetIntervalMs) { + // Emit explicit stall telemetry for field diagnosis. + RTC_LOG(LS_WARNING) << "iOS H264 encoder appears stalled. Forcing hard reset." + << " stalled_for_ms=" << (nowMs - _lastSuccessfulEncodeMs) + << " target_bps=" << _targetBitrateBps; + // Record reset time before reset call so recovery delay can be measured. + _lastHardResetMs = nowMs; + // Mark that we are awaiting first successful frame after forced reset. + _isInForcedRecovery = YES; + // Recreate compression session to recover from internal VideoToolbox stall. + [self resetCompressionSessionWithPixelFormat:[self pixelFormatOfFrame:frame]]; + } +#endif + BOOL isKeyframeRequired = NO; // Get a pixel buffer from the pool and copy frame data over. if ([self resetCompressionSessionIfNeededWithFrame:frame]) { @@ -949,6 +997,19 @@ - (void)frameWasEncoded:(OSStatus)status return; } + const int64_t nowMs = rtc::TimeMillis(); + // Successful output proves encoder is alive; refresh liveness timestamp. + _lastSuccessfulEncodeMs = nowMs; + // Log one-time recovery event after a forced reset once first frame arrives. + if (_isInForcedRecovery) { + const int64_t recoveryDelayMs = + _lastHardResetMs > 0 ? nowMs - _lastHardResetMs : -1; + RTC_LOG(LS_INFO) << "iOS H264 encoder recovered after forced hard reset." + << " recovery_delay_ms=" << recoveryDelayMs; + // Exit recovery mode after the first successful frame post-reset. + _isInForcedRecovery = NO; + } + BOOL isKeyframe = NO; CFArrayRef attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, 0); diff --git a/video/video_send_stream_impl.cc b/video/video_send_stream_impl.cc index dfcb9a30e9..637fd1812e 100644 --- a/video/video_send_stream_impl.cc +++ b/video/video_send_stream_impl.cc @@ -709,7 +709,17 @@ void VideoSendStreamImpl::SignalEncoderTimedOut() { // is supposed to, deregister as BitrateAllocatorObserver. This can happen // if a camera stops producing frames. if (encoder_target_rate_bps_ > 0) { - RTC_LOG(LS_INFO) << "SignalEncoderTimedOut, Encoder timed out."; + // Track this timeout instance for operational telemetry. + ++encoder_timeout_count_; + // Mark when this timeout began so we can compute recovery latency later. + encoder_timeout_started_ms_ = env_.clock().TimeInMilliseconds(); + // Emit a structured timeout log line with state needed for postmortems. + RTC_LOG(LS_INFO) << "SignalEncoderTimedOut, Encoder timed out." + << " timeout_count=" << encoder_timeout_count_ + << " target_rate_bps=" << encoder_target_rate_bps_ + << " running=" << IsRunning() + << " has_active_encodings=" << has_active_encodings_; + // Stop bitrate allocations while encoder is stalled to avoid wasted updates. bitrate_allocator_->RemoveObserver(this); } } @@ -773,7 +783,25 @@ void VideoSendStreamImpl::OnVideoLayersAllocationUpdated( void VideoSendStreamImpl::SignalEncoderActive() { RTC_DCHECK_RUN_ON(&thread_checker_); if (IsRunning()) { - RTC_LOG(LS_INFO) << "SignalEncoderActive, Encoder is active."; + // Capture "now" once so all derived telemetry values use one timestamp. + const int64_t now_ms = env_.clock().TimeInMilliseconds(); + // Compute how long the most recent timeout lasted, if any. + const int64_t timed_out_for_ms = + encoder_timeout_started_ms_ >= 0 ? now_ms - encoder_timeout_started_ms_ + : -1; + // Count this as a recovery only when we are transitioning from a timeout. + if (timed_out_for_ms >= 0) { + ++encoder_recovery_count_; + } + // Emit recovery telemetry that pairs with timeout logs above. + RTC_LOG(LS_INFO) << "SignalEncoderActive, Encoder is active." + << " recovery_count=" << encoder_recovery_count_ + << " timed_out_for_ms=" << timed_out_for_ms + << " target_rate_bps=" << encoder_target_rate_bps_ + << " running=" << IsRunning(); + // Clear timeout marker since encoder has resumed. + encoder_timeout_started_ms_ = -1; + // Re-attach bitrate observation after encoder activity resumes. bitrate_allocator_->AddObserver(this, GetAllocationConfig()); } } diff --git a/video/video_send_stream_impl.h b/video/video_send_stream_impl.h index fec8962c01..411f22deac 100644 --- a/video/video_send_stream_impl.h +++ b/video/video_send_stream_impl.h @@ -211,6 +211,12 @@ class VideoSendStreamImpl : public webrtc::VideoSendStream, std::atomic_bool activity_; bool timed_out_ RTC_GUARDED_BY(thread_checker_); + // Counts how many times this stream has entered encoder-timeout state. + uint32_t encoder_timeout_count_ RTC_GUARDED_BY(thread_checker_) = 0; + // Counts how many times the encoder became active after a timeout. + uint32_t encoder_recovery_count_ RTC_GUARDED_BY(thread_checker_) = 0; + // Stores wall-clock ms when timeout started; -1 means "not currently timed out". + int64_t encoder_timeout_started_ms_ RTC_GUARDED_BY(thread_checker_) = -1; BitrateAllocatorInterface* const bitrate_allocator_; From b1a3ce4a038e45f8bdb07d8e21dd1e694a22f087 Mon Sep 17 00:00:00 2001 From: Stream Bot Date: Mon, 6 Apr 2026 18:40:23 +0300 Subject: [PATCH 3/3] Fix simulcast reconfiguration after active layer increases --- .../video_stream_encoder_resource_manager.cc | 17 +++ .../video_stream_encoder_resource_manager.h | 6 + video/config/encoder_stream_factory.cc | 8 +- .../config/encoder_stream_factory_unittest.cc | 60 ++++++++- video/video_stream_encoder.cc | 16 +++ video/video_stream_encoder.h | 1 + video/video_stream_encoder_unittest.cc | 119 ++++++++++++++++++ 7 files changed, 222 insertions(+), 5 deletions(-) diff --git a/video/adaptation/video_stream_encoder_resource_manager.cc b/video/adaptation/video_stream_encoder_resource_manager.cc index ae575de690..1ce7c2d0b8 100644 --- a/video/adaptation/video_stream_encoder_resource_manager.cc +++ b/video/adaptation/video_stream_encoder_resource_manager.cc @@ -559,6 +559,23 @@ void VideoStreamEncoderResourceManager::UpdateBandwidthQualityScalerSettings( } } +void VideoStreamEncoderResourceManager::ResetAdaptationsForSimulcastChange() { + RTC_DCHECK_RUN_ON(encoder_queue_); + if (quality_scaler_resource_->is_started()) { + RTC_LOG(LS_INFO) << "Clearing quality scaler restrictions for simulcast " + "active-layer transition."; + quality_scaler_resource_->StopCheckForOveruse(); + RemoveResource(quality_scaler_resource_); + initial_frame_dropper_->OnQualityScalerSettingsUpdated(); + } + if (bandwidth_quality_scaler_resource_->is_started()) { + RTC_LOG(LS_INFO) << "Clearing bandwidth quality scaler restrictions for " + "simulcast active-layer transition."; + bandwidth_quality_scaler_resource_->StopCheckForOveruse(); + RemoveResource(bandwidth_quality_scaler_resource_); + } +} + void VideoStreamEncoderResourceManager::ConfigureQualityScaler( const VideoEncoder::EncoderInfo& encoder_info) { RTC_DCHECK_RUN_ON(encoder_queue_); diff --git a/video/adaptation/video_stream_encoder_resource_manager.h b/video/adaptation/video_stream_encoder_resource_manager.h index 1520bd5aef..aa9cf54562 100644 --- a/video/adaptation/video_stream_encoder_resource_manager.h +++ b/video/adaptation/video_stream_encoder_resource_manager.h @@ -109,6 +109,12 @@ class VideoStreamEncoderResourceManager // TODO(https://crbug.com/webrtc/11338): This can be made private if we // configure on SetDegredationPreference and SetEncoderSettings. void ConfigureQualityScaler(const VideoEncoder::EncoderInfo& encoder_info); + + // Stops the quality scaler and clears its accumulated adaptation + // restrictions. Called when the number of active simulcast layers increases + // from <=1 to >1, so that the source provides full-resolution frames for + // the new multi-layer configuration. + void ResetAdaptationsForSimulcastChange(); void ConfigureBandwidthQualityScaler( const VideoEncoder::EncoderInfo& encoder_info); diff --git a/video/config/encoder_stream_factory.cc b/video/config/encoder_stream_factory.cc index 558427478a..2baf10943b 100644 --- a/video/config/encoder_stream_factory.cc +++ b/video/config/encoder_stream_factory.cc @@ -69,13 +69,15 @@ bool IsTemporalLayersSupported(VideoCodecType codec_type) { } size_t FindRequiredActiveLayers(const VideoEncoderConfig& encoder_config) { - // Need enough layers so that at least the first active one is present. + // Need enough layers so that all active ones are present. + // Return the position after the highest active layer. + size_t highest = 0; for (size_t i = 0; i < encoder_config.number_of_streams; ++i) { if (encoder_config.simulcast_layers[i].active) { - return i + 1; + highest = i + 1; } } - return 0; + return highest; } // The selected thresholds for QVGA and VGA corresponded to a QP around 10. diff --git a/video/config/encoder_stream_factory_unittest.cc b/video/config/encoder_stream_factory_unittest.cc index a36efaab01..85f28a500c 100644 --- a/video/config/encoder_stream_factory_unittest.cc +++ b/video/config/encoder_stream_factory_unittest.cc @@ -340,12 +340,68 @@ TEST(EncoderStreamFactory, ReducesStreamCountWhenResolutionIsLow) { SizeIs(1)); } -TEST(EncoderStreamFactory, ReducesStreamCountDownToFirstActiveStream) { +TEST(EncoderStreamFactory, KeepsStreamCountToIncludeHighestActiveLayer) { EXPECT_THAT( CreateStreamResolutions({.number_of_streams = 3, .resolution = {.width = 100, .height = 100}, .first_active_layer_idx = 1}), - SizeIs(2)); + SizeIs(3)); +} + +TEST(EncoderStreamFactory, KeepsAllStreamsForSparseActivePattern) { + ExplicitKeyValueConfig field_trials(""); + VideoEncoderConfig encoder_config; + encoder_config.codec_type = VideoCodecType::kVideoCodecVP8; + encoder_config.number_of_streams = 3; + encoder_config.simulcast_layers.resize(3); + encoder_config.simulcast_layers[0].active = true; + encoder_config.simulcast_layers[1].active = false; + encoder_config.simulcast_layers[2].active = true; + auto streams = CreateEncoderStreams( + field_trials, {.width = 100, .height = 100}, encoder_config); + EXPECT_THAT(streams, SizeIs(3)); +} + +TEST(EncoderStreamFactory, KeepsStreamsForHighAndLowActive) { + ExplicitKeyValueConfig field_trials(""); + VideoEncoderConfig encoder_config; + encoder_config.codec_type = VideoCodecType::kVideoCodecVP8; + encoder_config.number_of_streams = 3; + encoder_config.simulcast_layers.resize(3); + encoder_config.simulcast_layers[0].active = true; + encoder_config.simulcast_layers[1].active = true; + encoder_config.simulcast_layers[2].active = false; + auto streams = CreateEncoderStreams( + field_trials, {.width = 100, .height = 100}, encoder_config); + EXPECT_THAT(streams, SizeIs(2)); +} + +TEST(EncoderStreamFactory, KeepsStreamsForOnlyHighestActive) { + ExplicitKeyValueConfig field_trials(""); + VideoEncoderConfig encoder_config; + encoder_config.codec_type = VideoCodecType::kVideoCodecVP8; + encoder_config.number_of_streams = 3; + encoder_config.simulcast_layers.resize(3); + encoder_config.simulcast_layers[0].active = false; + encoder_config.simulcast_layers[1].active = false; + encoder_config.simulcast_layers[2].active = true; + auto streams = CreateEncoderStreams( + field_trials, {.width = 100, .height = 100}, encoder_config); + EXPECT_THAT(streams, SizeIs(3)); +} + +TEST(EncoderStreamFactory, ReducesToOneStreamWhenOnlyLowestActive) { + ExplicitKeyValueConfig field_trials(""); + VideoEncoderConfig encoder_config; + encoder_config.codec_type = VideoCodecType::kVideoCodecVP8; + encoder_config.number_of_streams = 3; + encoder_config.simulcast_layers.resize(3); + encoder_config.simulcast_layers[0].active = true; + encoder_config.simulcast_layers[1].active = false; + encoder_config.simulcast_layers[2].active = false; + auto streams = CreateEncoderStreams( + field_trials, {.width = 100, .height = 100}, encoder_config); + EXPECT_THAT(streams, SizeIs(1)); } TEST(EncoderStreamFactory, diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index 5ddaf24f66..7d7257d6d3 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -1059,6 +1059,22 @@ void VideoStreamEncoder::ReconfigureEncoder() { AlignmentAdjuster::GetAlignmentAndMaybeAdjustScaleFactors( encoder_->GetEncoderInfo(), &encoder_config_, std::nullopt); + size_t num_active_layers = 0; + for (size_t i = 0; i < encoder_config_.number_of_streams; ++i) { + if (encoder_config_.simulcast_layers[i].active) { + ++num_active_layers; + } + } + if (num_active_layers > prev_num_active_simulcast_layers_ && + prev_num_active_simulcast_layers_ > 0) { + RTC_LOG(LS_INFO) << "Active simulcast layers increased from " + << prev_num_active_simulcast_layers_ << " to " + << num_active_layers + << ". Resetting quality-scaler restrictions."; + stream_resource_manager_.ResetAdaptationsForSimulcastChange(); + } + prev_num_active_simulcast_layers_ = num_active_layers; + std::vector streams; if (encoder_config_.video_stream_factory) { // Note: only tests set their own EncoderStreamFactory... diff --git a/video/video_stream_encoder.h b/video/video_stream_encoder.h index 917d928149..17fbca045e 100644 --- a/video/video_stream_encoder.h +++ b/video/video_stream_encoder.h @@ -313,6 +313,7 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, std::optional last_frame_info_ RTC_GUARDED_BY(encoder_queue_); int crop_width_ RTC_GUARDED_BY(encoder_queue_) = 0; int crop_height_ RTC_GUARDED_BY(encoder_queue_) = 0; + size_t prev_num_active_simulcast_layers_ RTC_GUARDED_BY(encoder_queue_) = 0; std::optional encoder_target_bitrate_bps_ RTC_GUARDED_BY(encoder_queue_); size_t max_data_payload_length_ RTC_GUARDED_BY(encoder_queue_) = 0; diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc index 17f4ea1a2c..63664670a7 100644 --- a/video/video_stream_encoder_unittest.cc +++ b/video/video_stream_encoder_unittest.cc @@ -6815,6 +6815,125 @@ TEST_F(VideoStreamEncoderTest, video_stream_encoder_->Stop(); } +TEST_F(VideoStreamEncoderTest, + QualityScalerRestrictionsResetWhenActiveLayersIncrease) { + // Set up 3-stream simulcast with only the highest layer active (1:1 call). + ResetEncoder("VP8", 3, 1, 1, false); + fake_encoder_.SetQualityScaling(true); + const int kWidth = 1280; + const int kHeight = 720; + + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( + kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate, + 0, 0, 0); + + // Send a frame and trigger quality low to accumulate a resolution adaptation. + video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight)); + WaitForEncodedFrame(1); + video_stream_encoder_->TriggerQualityLow(); + + video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight)); + WaitForEncodedFrame(2); + + EXPECT_THAT(video_source_.sink_wants(), WantsMaxPixels(Lt(kWidth * kHeight))); + + // Now transition to multi-layer: activate all 3 layers (3rd participant). + VideoEncoderConfig video_encoder_config; + test::FillEncoderConfiguration(PayloadStringToCodecType("VP8"), 3, + &video_encoder_config); + video_encoder_config.video_stream_factory = nullptr; + for (auto& layer : video_encoder_config.simulcast_layers) { + layer.num_temporal_layers = 1; + layer.max_framerate = kDefaultFramerate; + } + video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps(); + video_encoder_config.content_type = + VideoEncoderConfig::ContentType::kRealtimeVideo; + video_encoder_config.simulcast_layers[0].active = true; + video_encoder_config.simulcast_layers[1].active = true; + video_encoder_config.simulcast_layers[2].active = true; + + video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(), + kMaxPayloadLength); + video_stream_encoder_->WaitUntilTaskQueueIsIdle(); + AdvanceTime(TimeDelta::Zero()); + + // Quality scaler restrictions should have been cleared on the active-layer + // increase so the source can provide full-resolution frames. + EXPECT_THAT(video_source_.sink_wants(), ResolutionMax()); + + video_stream_encoder_->Stop(); +} + +TEST_F(VideoStreamEncoderTest, + QualityScalerRestrictionsResetOnTwoToThreeLayerIncrease) { + // Set up 3-stream simulcast with 2 layers active. + ResetEncoder("VP8", 3, 1, 1, false); + fake_encoder_.SetQualityScaling(true); + const int kWidth = 1280; + const int kHeight = 720; + + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( + kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate, + 0, 0, 0); + + // Start with 2 active layers. + VideoEncoderConfig config_2_active; + test::FillEncoderConfiguration(PayloadStringToCodecType("VP8"), 3, + &config_2_active); + config_2_active.video_stream_factory = nullptr; + for (auto& layer : config_2_active.simulcast_layers) { + layer.num_temporal_layers = 1; + layer.max_framerate = kDefaultFramerate; + } + config_2_active.max_bitrate_bps = kSimulcastTargetBitrate.bps(); + config_2_active.content_type = + VideoEncoderConfig::ContentType::kRealtimeVideo; + config_2_active.simulcast_layers[0].active = true; + config_2_active.simulcast_layers[1].active = true; + config_2_active.simulcast_layers[2].active = false; + + video_stream_encoder_->ConfigureEncoder(config_2_active.Copy(), + kMaxPayloadLength); + video_stream_encoder_->WaitUntilTaskQueueIsIdle(); + + // Send a frame and trigger quality low to accumulate a restriction. + video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight)); + WaitForEncodedFrame(1); + video_stream_encoder_->TriggerQualityLow(); + + video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight)); + WaitForEncodedFrame(2); + + EXPECT_THAT(video_source_.sink_wants(), WantsMaxPixels(Lt(kWidth * kHeight))); + + // Now increase to 3 active layers. + VideoEncoderConfig config_3_active; + test::FillEncoderConfiguration(PayloadStringToCodecType("VP8"), 3, + &config_3_active); + config_3_active.video_stream_factory = nullptr; + for (auto& layer : config_3_active.simulcast_layers) { + layer.num_temporal_layers = 1; + layer.max_framerate = kDefaultFramerate; + } + config_3_active.max_bitrate_bps = kSimulcastTargetBitrate.bps(); + config_3_active.content_type = + VideoEncoderConfig::ContentType::kRealtimeVideo; + config_3_active.simulcast_layers[0].active = true; + config_3_active.simulcast_layers[1].active = true; + config_3_active.simulcast_layers[2].active = true; + + video_stream_encoder_->ConfigureEncoder(config_3_active.Copy(), + kMaxPayloadLength); + video_stream_encoder_->WaitUntilTaskQueueIsIdle(); + AdvanceTime(TimeDelta::Zero()); + + // Restrictions should be cleared on the 2 -> 3 active-layer increase. + EXPECT_THAT(video_source_.sink_wants(), ResolutionMax()); + + video_stream_encoder_->Stop(); +} + TEST_F(VideoStreamEncoderTest, ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) { const int kTooSmallWidth = 10;