From 85a34a5967d57480af8527e1674a9091575d7a29 Mon Sep 17 00:00:00 2001 From: David Liu Date: Sat, 23 Oct 2021 15:33:55 +0900 Subject: [PATCH] Use fallback encoder if primary can't be created In case if primary encoder can't be instantiated (max number of instances has reached, for example), use fallback encoder. Originally from: https://webrtc.googlesource.com/src/+/9b2a7461f099e3b732993925b24fd4752fd9c9a3 --- media/engine/simulcast_encoder_adapter.cc | 41 ++++++++++++---- .../simulcast_encoder_adapter_unittest.cc | 49 +++++++++++++++++++ 2 files changed, 80 insertions(+), 10 deletions(-) diff --git a/media/engine/simulcast_encoder_adapter.cc b/media/engine/simulcast_encoder_adapter.cc index bee7b23c4e..832c0bde9f 100644 --- a/media/engine/simulcast_encoder_adapter.cc +++ b/media/engine/simulcast_encoder_adapter.cc @@ -684,15 +684,15 @@ void SimulcastEncoderAdapter::DestroyStoredEncoders() { } } + std::unique_ptr SimulcastEncoderAdapter::FetchOrCreateEncoderContext( bool is_lowest_quality_stream) { bool prefer_temporal_support = fallback_encoder_factory_ != nullptr && is_lowest_quality_stream && prefer_temporal_support_on_base_layer_; - - // Toggling of |prefer_temporal_support| requires encoder recreation. Find - // and reuse encoder with desired |prefer_temporal_support|. Otherwise, if + // Toggling of `prefer_temporal_support` requires encoder recreation. Find + // and reuse encoder with desired `prefer_temporal_support`. Otherwise, if // there is no such encoder in the cache, create a new instance. auto encoder_context_iter = std::find_if(cached_encoder_contexts_.begin(), @@ -700,24 +700,45 @@ SimulcastEncoderAdapter::FetchOrCreateEncoderContext( return encoder_context->prefer_temporal_support() == prefer_temporal_support; }); - std::unique_ptr encoder_context; if (encoder_context_iter != cached_encoder_contexts_.end()) { encoder_context = std::move(*encoder_context_iter); cached_encoder_contexts_.erase(encoder_context_iter); } else { - std::unique_ptr encoder = + std::unique_ptr primary_encoder = primary_encoder_factory_->CreateVideoEncoder(video_format_); + std::unique_ptr fallback_encoder; if (fallback_encoder_factory_ != nullptr) { - encoder = CreateVideoEncoderSoftwareFallbackWrapper( - fallback_encoder_factory_->CreateVideoEncoder(video_format_), - std::move(encoder), prefer_temporal_support); + fallback_encoder = + fallback_encoder_factory_->CreateVideoEncoder(video_format_); + } + std::unique_ptr encoder; + VideoEncoder::EncoderInfo primary_info; + VideoEncoder::EncoderInfo fallback_info; + if (primary_encoder != nullptr) { + primary_info = primary_encoder->GetEncoderInfo(); + fallback_info = primary_info; + if (fallback_encoder == nullptr) { + encoder = std::move(primary_encoder); + } else { + encoder = CreateVideoEncoderSoftwareFallbackWrapper( + std::move(fallback_encoder), std::move(primary_encoder), + prefer_temporal_support); + } + } else if (fallback_encoder != nullptr) { + RTC_LOG(LS_WARNING) << "Failed to create primary " << video_format_.name + << " encoder. Use fallback encoder."; + fallback_info = fallback_encoder->GetEncoderInfo(); + primary_info = fallback_info; + encoder = std::move(fallback_encoder); + } else { + RTC_LOG(LS_ERROR) << "Failed to create primary and fallback " + << video_format_.name << " encoders."; + return nullptr; } - encoder_context = std::make_unique( std::move(encoder), prefer_temporal_support); } - encoder_context->encoder().RegisterEncodeCompleteCallback( encoded_complete_callback_); return encoder_context; diff --git a/media/engine/simulcast_encoder_adapter_unittest.cc b/media/engine/simulcast_encoder_adapter_unittest.cc index a74a2c3785..65c2b5dbcb 100644 --- a/media/engine/simulcast_encoder_adapter_unittest.cc +++ b/media/engine/simulcast_encoder_adapter_unittest.cc @@ -171,6 +171,9 @@ class MockVideoEncoderFactory : public VideoEncoderFactory { const std::vector& encoders() const; void SetEncoderNames(const std::vector& encoder_names); + void set_create_video_encode_return_nullptr(bool return_nullptr) { + create_video_encoder_return_nullptr_ = return_nullptr; + } void set_init_encode_return_value(int32_t value); void set_requested_resolution_alignments( std::vector requested_resolution_alignments) { @@ -183,6 +186,7 @@ class MockVideoEncoderFactory : public VideoEncoderFactory { void DestroyVideoEncoder(VideoEncoder* encoder); private: + bool create_video_encoder_return_nullptr_ = false; int32_t init_encode_return_value_ = 0; std::vector encoders_; std::vector encoder_names_; @@ -340,6 +344,10 @@ std::vector MockVideoEncoderFactory::GetSupportedFormats() std::unique_ptr MockVideoEncoderFactory::CreateVideoEncoder( const SdpVideoFormat& format) { + if (create_video_encoder_return_nullptr_) { + return nullptr; + } + auto encoder = std::make_unique<::testing::NiceMock>(this); encoder->set_init_encode_return_value(init_encode_return_value_); const char* encoder_name = encoder_names_.empty() @@ -1683,5 +1691,46 @@ TEST_F(TestSimulcastEncoderAdapterFake, EXPECT_NE(helper_->factory()->encoders()[0], prev_encoder); } +TEST_F(TestSimulcastEncoderAdapterFake, + UseFallbackEncoderIfCreatePrimaryEncoderFailed) { + // Enable support for fallback encoder factory and re-setup. + use_fallback_factory_ = true; + SetUp(); + SimulcastTestFixtureImpl::DefaultSettings( + &codec_, static_cast(kTestTemporalLayerProfile), + kVideoCodecVP8); + codec_.numberOfSimulcastStreams = 1; + helper_->factory()->SetEncoderNames({"primary"}); + helper_->fallback_factory()->SetEncoderNames({"fallback"}); + + // Emulate failure at creating of primary encoder and verify that SEA switches + // to fallback encoder. + helper_->factory()->set_create_video_encode_return_nullptr(true); + EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); + ASSERT_EQ(0u, helper_->factory()->encoders().size()); + ASSERT_EQ(1u, helper_->fallback_factory()->encoders().size()); + EXPECT_EQ("fallback", adapter_->GetEncoderInfo().implementation_name); +} + +TEST_F(TestSimulcastEncoderAdapterFake, + InitEncodeReturnsErrorIfEncoderCannotBeCreated) { + // Enable support for fallback encoder factory and re-setup. + use_fallback_factory_ = true; + SetUp(); + SimulcastTestFixtureImpl::DefaultSettings( + &codec_, static_cast(kTestTemporalLayerProfile), + kVideoCodecVP8); + codec_.numberOfSimulcastStreams = 1; + helper_->factory()->SetEncoderNames({"primary"}); + helper_->fallback_factory()->SetEncoderNames({"fallback"}); + + // Emulate failure at creating of primary and fallback encoders and verify + // that `InitEncode` returns an error. + helper_->factory()->set_create_video_encode_return_nullptr(true); + helper_->fallback_factory()->set_create_video_encode_return_nullptr(true); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_MEMORY, + adapter_->InitEncode(&codec_, kSettings)); +} + } // namespace test } // namespace webrtc