From dab41142bf2d392448afaff0b36cdcffc76ac3c8 Mon Sep 17 00:00:00 2001 From: Cameron Gutman Date: Mon, 2 Jan 2023 18:48:30 -0600 Subject: [PATCH 1/2] Add Media Foundation encoder --- docs/source/about/advanced_usage.rst | 1 + src/video.cpp | 40 ++++++++++++++++++++++++ src_assets/common/assets/web/config.html | 1 + 3 files changed, 42 insertions(+) diff --git a/docs/source/about/advanced_usage.rst b/docs/source/about/advanced_usage.rst index cf4a348c184..e2162676f36 100644 --- a/docs/source/about/advanced_usage.rst +++ b/docs/source/about/advanced_usage.rst @@ -822,6 +822,7 @@ encoder nvenc For NVIDIA graphics cards quicksync For Intel graphics cards amdvce For AMD graphics cards + mf Media Foundation (slower than encoders above) software Encoding occurs on the CPU ========= =========== diff --git a/src/video.cpp b/src/video.cpp index 1b8b682fb3f..6523e003ef6 100644 --- a/src/video.cpp +++ b/src/video.cpp @@ -599,6 +599,45 @@ namespace video { PARALLEL_ENCODING, dxgi_make_hwdevice_ctx }; + + static encoder_t mfenc { + "mf"sv, + AV_HWDEVICE_TYPE_NONE, AV_HWDEVICE_TYPE_NONE, + AV_PIX_FMT_NONE, + AV_PIX_FMT_NV12, AV_PIX_FMT_P010, + { + // Common options + { + { "hw_encoding"s, 1 }, + { "scenario"s, "display_remoting"s }, + { "low_latency", 1 }, + + // Other rate controls exist but they are inconsistently implemented + // and can cause codec init to totally fail if unsupported. + { "rate_control"s, "cbr"s }, + }, + {}, // SDR-specific options + {}, // HDR-specific options + std::nullopt, + "hevc_mf"s, + }, + { + // Common options + { + { "hw_encoding"s, 1 }, + { "scenario"s, "display_remoting"s }, + { "low_latency", 1 }, + { "rate_control"s, "cbr"s }, + }, + {}, // SDR-specific options + {}, // HDR-specific options + std::nullopt, + "h264_mf"s, + }, + PARALLEL_ENCODING, + nullptr + }; + #endif static encoder_t software { @@ -717,6 +756,7 @@ namespace video { #ifdef _WIN32 &quicksync, &amdvce, + &mfenc, #endif #ifdef __linux__ &vaapi, diff --git a/src_assets/common/assets/web/config.html b/src_assets/common/assets/web/config.html index 3572dd70f84..5d400d2f1ef 100644 --- a/src_assets/common/assets/web/config.html +++ b/src_assets/common/assets/web/config.html @@ -671,6 +671,7 @@

Configuration

+
From cf643120757653be3d68351f1cd7a5923e1e88c7 Mon Sep 17 00:00:00 2001 From: Cameron Gutman Date: Wed, 18 Jan 2023 20:51:33 -0600 Subject: [PATCH 2/2] Use GPU accelerated RGB->YUV conversion for Media Foundation --- src/platform/common.h | 2 +- src/platform/linux/cuda.cpp | 6 +-- src/platform/linux/vaapi.cpp | 2 +- src/platform/macos/nv12_zero_device.cpp | 2 +- src/platform/macos/nv12_zero_device.h | 2 +- src/platform/windows/display_vram.cpp | 33 ++++++++++++- src/video.cpp | 63 ++++++++++++++++--------- 7 files changed, 79 insertions(+), 31 deletions(-) diff --git a/src/platform/common.h b/src/platform/common.h index fa8104dcf64..259249ea01a 100644 --- a/src/platform/common.h +++ b/src/platform/common.h @@ -216,7 +216,7 @@ namespace platf { * implementations must take ownership of 'frame' */ virtual int - set_frame(AVFrame *frame, AVBufferRef *hw_frames_ctx) { + set_frame(AVFrame *frame, AVBufferRef *hw_frames_ctx, int target_format) { BOOST_LOG(error) << "Illegal call to hwdevice_t::set_frame(). Did you forget to override it?"; return -1; }; diff --git a/src/platform/linux/cuda.cpp b/src/platform/linux/cuda.cpp index 55a4a47f858..42bfd18f0ea 100644 --- a/src/platform/linux/cuda.cpp +++ b/src/platform/linux/cuda.cpp @@ -102,7 +102,7 @@ namespace cuda { } int - set_frame(AVFrame *frame, AVBufferRef *hw_frames_ctx) override { + set_frame(AVFrame *frame, AVBufferRef *hw_frames_ctx, int target_format) override { this->hwframe.reset(frame); this->frame = frame; @@ -193,8 +193,8 @@ namespace cuda { } int - set_frame(AVFrame *frame, AVBufferRef *hw_frames_ctx) { - if (cuda_t::set_frame(frame, hw_frames_ctx)) { + set_frame(AVFrame *frame, AVBufferRef *hw_frames_ctx, int target_format) { + if (cuda_t::set_frame(frame, hw_frames_ctx, target_format)) { return -1; } diff --git a/src/platform/linux/vaapi.cpp b/src/platform/linux/vaapi.cpp index 12281ac4292..6102f42a76d 100644 --- a/src/platform/linux/vaapi.cpp +++ b/src/platform/linux/vaapi.cpp @@ -329,7 +329,7 @@ namespace va { } int - set_frame(AVFrame *frame, AVBufferRef *hw_frames_ctx) override { + set_frame(AVFrame *frame, AVBufferRef *hw_frames_ctx, int target_format) override { this->hwframe.reset(frame); this->frame = frame; diff --git a/src/platform/macos/nv12_zero_device.cpp b/src/platform/macos/nv12_zero_device.cpp index 0c62328f762..c0254a16025 100644 --- a/src/platform/macos/nv12_zero_device.cpp +++ b/src/platform/macos/nv12_zero_device.cpp @@ -56,7 +56,7 @@ namespace platf { } int - nv12_zero_device::set_frame(AVFrame *frame, AVBufferRef *hw_frames_ctx) { + nv12_zero_device::set_frame(AVFrame *frame, AVBufferRef *hw_frames_ctx, int target_format) { this->frame = frame; av_frame.reset(frame); diff --git a/src/platform/macos/nv12_zero_device.h b/src/platform/macos/nv12_zero_device.h index 53b6eff19d7..a17726df797 100644 --- a/src/platform/macos/nv12_zero_device.h +++ b/src/platform/macos/nv12_zero_device.h @@ -23,7 +23,7 @@ namespace platf { int convert(img_t &img); int - set_frame(AVFrame *frame, AVBufferRef *hw_frames_ctx); + set_frame(AVFrame *frame, AVBufferRef *hw_frames_ctx, int target_format); void set_colorspace(std::uint32_t colorspace, std::uint32_t color_range); }; diff --git a/src/platform/windows/display_vram.cpp b/src/platform/windows/display_vram.cpp index fa3ecf4df4c..a976d1c6e64 100644 --- a/src/platform/windows/display_vram.cpp +++ b/src/platform/windows/display_vram.cpp @@ -355,6 +355,16 @@ namespace platf::dxgi { ID3D11ShaderResourceView *emptyShaderResourceView = nullptr; device_ctx->PSSetShaderResources(0, 1, &emptyShaderResourceView); + // If this requires conversion to a swframe, transfer data back from the GPU + if (swframe) { + auto status = av_hwframe_transfer_data(this->frame, hwframe.get(), 0); + if (status < 0) { + char string[AV_ERROR_MAX_STRING_SIZE]; + BOOST_LOG(error) << "Failed to transfer image data from hardware frame: "sv << av_make_error_string(string, AV_ERROR_MAX_STRING_SIZE, status); + return -1; + } + } + return 0; } @@ -425,9 +435,25 @@ namespace platf::dxgi { } int - set_frame(AVFrame *frame, AVBufferRef *hw_frames_ctx) override { + set_frame(AVFrame *frame, AVBufferRef *hw_frames_ctx, int target_format) override { this->hwframe.reset(frame); - this->frame = frame; + + // If we have to do conversion to a different swframe pixel format, set up our context to do so + if (frame->format != target_format) { + swframe.reset(av_frame_alloc()); + swframe->format = target_format; + swframe->width = frame->width; + swframe->height = frame->height; + + // Allocate buffers for the swframe data + av_frame_get_buffer(swframe.get(), 0); + + // Expose the swframe to the encoder rather than the hwframe + this->frame = swframe.get(); + } + else { + this->frame = frame; + } // Populate this frame with a hardware buffer if one isn't there already if (!frame->buf[0]) { @@ -724,6 +750,9 @@ namespace platf::dxgi { public: frame_t hwframe; + // Used when target_format differs from hwframe.format + frame_t swframe; + ::video::color_t *color_p; buf_t info_scene; diff --git a/src/video.cpp b/src/video.cpp index 6523e003ef6..bf8c6f8196d 100644 --- a/src/video.cpp +++ b/src/video.cpp @@ -141,7 +141,7 @@ namespace video { } int - set_frame(AVFrame *frame, AVBufferRef *hw_frames_ctx) { + set_frame(AVFrame *frame, AVBufferRef *hw_frames_ctx, int target_format) { this->frame = frame; // If it's a hwframe, allocate buffers for hardware @@ -321,6 +321,12 @@ namespace video { AVHWDeviceType base_dev_type, derived_dev_type; AVPixelFormat dev_pix_fmt; + // Encoder requires frames to be read back into the sw_pix_fmt + // prior to being passed to the encoder. This can provide GPU + // accelerated RGB->YUV color conversion for encoders that can't + // otherwise accept GPU surfaces as input. + bool sw_frame_readback; + AVPixelFormat static_pix_fmt, dynamic_pix_fmt; struct { @@ -445,10 +451,10 @@ namespace video { "nvenc"sv, #ifdef _WIN32 AV_HWDEVICE_TYPE_D3D11VA, AV_HWDEVICE_TYPE_NONE, - AV_PIX_FMT_D3D11, + AV_PIX_FMT_D3D11, false, #else AV_HWDEVICE_TYPE_CUDA, AV_HWDEVICE_TYPE_NONE, - AV_PIX_FMT_CUDA, + AV_PIX_FMT_CUDA, false, #endif AV_PIX_FMT_NV12, AV_PIX_FMT_P010, { @@ -504,6 +510,7 @@ namespace video { AV_HWDEVICE_TYPE_D3D11VA, AV_HWDEVICE_TYPE_QSV, AV_PIX_FMT_QSV, + false, AV_PIX_FMT_NV12, AV_PIX_FMT_P010, { @@ -557,7 +564,7 @@ namespace video { static encoder_t amdvce { "amdvce"sv, AV_HWDEVICE_TYPE_D3D11VA, AV_HWDEVICE_TYPE_NONE, - AV_PIX_FMT_D3D11, + AV_PIX_FMT_D3D11, false, AV_PIX_FMT_NV12, AV_PIX_FMT_P010, { // Common options @@ -602,8 +609,8 @@ namespace video { static encoder_t mfenc { "mf"sv, - AV_HWDEVICE_TYPE_NONE, AV_HWDEVICE_TYPE_NONE, - AV_PIX_FMT_NONE, + AV_HWDEVICE_TYPE_D3D11VA, AV_HWDEVICE_TYPE_NONE, + AV_PIX_FMT_D3D11, true, AV_PIX_FMT_NV12, AV_PIX_FMT_P010, { // Common options @@ -635,7 +642,7 @@ namespace video { "h264_mf"s, }, PARALLEL_ENCODING, - nullptr + dxgi_make_hwdevice_ctx }; #endif @@ -643,7 +650,7 @@ namespace video { static encoder_t software { "software"sv, AV_HWDEVICE_TYPE_NONE, AV_HWDEVICE_TYPE_NONE, - AV_PIX_FMT_NONE, + AV_PIX_FMT_NONE, false, AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUV420P10, { // x265's Info SEI is so long that it causes the IDR picture data to be @@ -681,7 +688,7 @@ namespace video { static encoder_t vaapi { "vaapi"sv, AV_HWDEVICE_TYPE_VAAPI, AV_HWDEVICE_TYPE_NONE, - AV_PIX_FMT_VAAPI, + AV_PIX_FMT_VAAPI, false, AV_PIX_FMT_NV12, AV_PIX_FMT_YUV420P10, { // Common options @@ -717,7 +724,7 @@ namespace video { static encoder_t videotoolbox { "videotoolbox"sv, AV_HWDEVICE_TYPE_NONE, AV_HWDEVICE_TYPE_NONE, - AV_PIX_FMT_VIDEOTOOLBOX, + AV_PIX_FMT_VIDEOTOOLBOX, false, AV_PIX_FMT_NV12, AV_PIX_FMT_NV12, { // Common options @@ -756,7 +763,7 @@ namespace video { #ifdef _WIN32 &quicksync, &amdvce, - &mfenc, + &mfenc, // Should be lowest priority hw encoder #endif #ifdef __linux__ &vaapi, @@ -1270,16 +1277,6 @@ namespace video { return std::nullopt; } - if (auto status = avcodec_open2(ctx.get(), codec, &options)) { - char err_str[AV_ERROR_MAX_STRING_SIZE] { 0 }; - BOOST_LOG(error) - << "Could not open codec ["sv - << video_format.name << "]: "sv - << av_make_error_string(err_str, AV_ERROR_MAX_STRING_SIZE, status); - - return std::nullopt; - } - frame_t frame { av_frame_alloc() }; frame->format = ctx->pix_fmt; frame->width = ctx->width; @@ -1331,12 +1328,34 @@ namespace video { device = std::move(hwdevice); } - if (device->set_frame(frame.release(), ctx->hw_frames_ctx)) { + // If this encoder doesn't accept hardware frames, use the swformat + if (hardware && encoder.sw_frame_readback) { + ctx->pix_fmt = sw_fmt; + } + + if (device->set_frame(frame.release(), ctx->hw_frames_ctx, ctx->pix_fmt)) { return std::nullopt; } device->set_colorspace(sws_color_space, ctx->color_range); + // FFmpeg doesn't allow a hw_frames_ctx to be attached to the codec context + // if we're going to be using a software pixel format, so we will remove it + // now that set_frame() has had a chance to attach it to the AVFrame. + if (hardware && encoder.sw_frame_readback) { + av_buffer_unref(&ctx->hw_frames_ctx); + } + + if (auto status = avcodec_open2(ctx.get(), codec, &options)) { + char err_str[AV_ERROR_MAX_STRING_SIZE] { 0 }; + BOOST_LOG(error) + << "Could not open codec ["sv + << video_format.name << "]: "sv + << av_make_error_string(err_str, AV_ERROR_MAX_STRING_SIZE, status); + + return std::nullopt; + } + session_t session { std::move(ctx), std::move(device),