Various shader-related improvements

cmake/compile_definitions/windows.cmake

@@ -45,6 +45,7 @@ set(PLATFORM_TARGET_FILES
         src/platform/windows/display_base.cpp
         src/platform/windows/display_vram.cpp
         src/platform/windows/display_ram.cpp
+        src/platform/windows/dx_helpers.cpp
         src/platform/windows/audio.cpp
         third-party/ViGEmClient/src/ViGEmClient.cpp
         third-party/ViGEmClient/include/ViGEm/Client.h

src/nvenc/nvenc_base.cpp

@@ -156,7 +156,7 @@ namespace nvenc {
     };
 
     auto buffer_is_yuv444 = [&]() {
-      return buffer_format == NV_ENC_BUFFER_FORMAT_YUV444 || buffer_format == NV_ENC_BUFFER_FORMAT_YUV444_10BIT;
+      return buffer_format == NV_ENC_BUFFER_FORMAT_AYUV || buffer_format == NV_ENC_BUFFER_FORMAT_YUV444_10BIT;
     };
 
     {
@@ -216,8 +216,6 @@ namespace nvenc {
     enc_config.rcParams.multiPass = config.two_pass == nvenc_two_pass::quarter_resolution ? NV_ENC_TWO_PASS_QUARTER_RESOLUTION :
                                     config.two_pass == nvenc_two_pass::full_resolution    ? NV_ENC_TWO_PASS_FULL_RESOLUTION :
                                                                                             NV_ENC_MULTI_PASS_DISABLED;
-
-    enc_config.rcParams.enableAQ = config.adaptive_quantization;
     enc_config.rcParams.averageBitRate = client_config.bitrate * 1000;
 
     if (get_encoder_cap(NV_ENC_CAPS_SUPPORT_CUSTOM_VBV_BUF_SIZE)) {
@@ -308,7 +306,9 @@ namespace nvenc {
         auto &format_config = enc_config.encodeCodecConfig.av1Config;
         format_config.repeatSeqHdr = 1;
         format_config.idrPeriod = NVENC_INFINITE_GOPLENGTH;
-        format_config.chromaFormatIDC = 1;  // YUV444 not supported by NVENC yet
+        if (buffer_is_yuv444()) {
+          format_config.chromaFormatIDC = 3;
+        }
         format_config.enableBitstreamPadding = config.insert_filler_data;
         if (buffer_is_10bit()) {
           format_config.inputPixelBitDepthMinus8 = 2;

src/nvenc/nvenc_d3d11.cpp

@@ -68,6 +68,7 @@ namespace nvenc {
       D3D11_TEXTURE2D_DESC desc = {};
       desc.Width = encoder_params.width;
       desc.Height = encoder_params.height;
+      if (encoder_params.buffer_format == NV_ENC_BUFFER_FORMAT_YUV444_10BIT) desc.Height *= 3;
       desc.MipLevels = 1;
       desc.ArraySize = 1;
       desc.Format = dxgi_format_from_nvenc_format(encoder_params.buffer_format);

src/nvenc/nvenc_utils.cpp

@@ -12,6 +12,12 @@ namespace nvenc {
       case NV_ENC_BUFFER_FORMAT_NV12:
         return DXGI_FORMAT_NV12;
 
+      case NV_ENC_BUFFER_FORMAT_AYUV:
+        return DXGI_FORMAT_AYUV;
+
+      case NV_ENC_BUFFER_FORMAT_YUV444_10BIT:
+        return DXGI_FORMAT_R16_UNORM;
+
       default:
         return DXGI_FORMAT_UNKNOWN;
     }
@@ -27,6 +33,12 @@ namespace nvenc {
       case platf::pix_fmt_e::p010:
         return NV_ENC_BUFFER_FORMAT_YUV420_10BIT;
 
+      case platf::pix_fmt_e::ayuv:
+        return NV_ENC_BUFFER_FORMAT_AYUV;
+
+      case platf::pix_fmt_e::yuv10_planar:
+        return NV_ENC_BUFFER_FORMAT_YUV444_10BIT;
+
       default:
         return NV_ENC_BUFFER_FORMAT_UNDEFINED;
     }

src/platform/common.h

@@ -196,6 +196,8 @@ namespace platf {
     yuv420p10,
     nv12,
     p010,
+    ayuv,
+    yuv10_planar,
     unknown
   };
 
@@ -210,6 +212,8 @@ namespace platf {
       _CONVERT(yuv420p10);
       _CONVERT(nv12);
       _CONVERT(p010);
+      _CONVERT(ayuv);
+      _CONVERT(yuv10_planar);
       _CONVERT(unknown);
     }
 #undef _CONVERT

src/platform/windows/display.h

@@ -11,6 +11,8 @@
 #include <dxgi.h>
 #include <dxgi1_6.h>
 
+#include <comdef.h>
+
 #include "src/platform/common.h"
 #include "src/utility.h"
 #include "src/video.h"
@@ -28,6 +30,10 @@ namespace platf::dxgi {
     dxgi->Release();
   }
 
+  _COM_SMARTPTR_TYPEDEF(ID3DBlob, IID_ID3DBlob);
+  _COM_SMARTPTR_TYPEDEF(ID3D11VertexShader, IID_ID3D11VertexShader);
+  _COM_SMARTPTR_TYPEDEF(ID3D11PixelShader, IID_ID3D11PixelShader);
+
   using factory1_t = util::safe_ptr<IDXGIFactory1, Release<IDXGIFactory1>>;
   using dxgi_t = util::safe_ptr<IDXGIDevice, Release<IDXGIDevice>>;
   using dxgi1_t = util::safe_ptr<IDXGIDevice1, Release<IDXGIDevice1>>;
@@ -45,16 +51,13 @@ namespace platf::dxgi {
   using resource_t = util::safe_ptr<IDXGIResource, Release<IDXGIResource>>;
   using resource1_t = util::safe_ptr<IDXGIResource1, Release<IDXGIResource1>>;
   using multithread_t = util::safe_ptr<ID3D11Multithread, Release<ID3D11Multithread>>;
-  using vs_t = util::safe_ptr<ID3D11VertexShader, Release<ID3D11VertexShader>>;
-  using ps_t = util::safe_ptr<ID3D11PixelShader, Release<ID3D11PixelShader>>;
   using blend_t = util::safe_ptr<ID3D11BlendState, Release<ID3D11BlendState>>;
   using input_layout_t = util::safe_ptr<ID3D11InputLayout, Release<ID3D11InputLayout>>;
   using render_target_t = util::safe_ptr<ID3D11RenderTargetView, Release<ID3D11RenderTargetView>>;
   using shader_res_t = util::safe_ptr<ID3D11ShaderResourceView, Release<ID3D11ShaderResourceView>>;
   using buf_t = util::safe_ptr<ID3D11Buffer, Release<ID3D11Buffer>>;
   using raster_state_t = util::safe_ptr<ID3D11RasterizerState, Release<ID3D11RasterizerState>>;
   using sampler_state_t = util::safe_ptr<ID3D11SamplerState, Release<ID3D11SamplerState>>;
-  using blob_t = util::safe_ptr<ID3DBlob, Release<ID3DBlob>>;
   using depth_stencil_state_t = util::safe_ptr<ID3D11DepthStencilState, Release<ID3D11DepthStencilState>>;
   using depth_stencil_view_t = util::safe_ptr<ID3D11DepthStencilView, Release<ID3D11DepthStencilView>>;
   using keyed_mutex_t = util::safe_ptr<IDXGIKeyedMutex, Release<IDXGIKeyedMutex>>;
@@ -329,8 +332,8 @@ namespace platf::dxgi {
     blend_t blend_invert;
     blend_t blend_disable;
 
-    ps_t cursor_ps;
-    vs_t cursor_vs;
+    ID3D11VertexShaderPtr cursor_vs;
+    ID3D11PixelShaderPtr cursor_ps;
 
     gpu_cursor_t cursor_alpha;
     gpu_cursor_t cursor_xor;