Implement HDR support for Windows

docs/source/about/usage.rst

@@ -257,6 +257,18 @@ Considerations
   instead it simply starts a stream.
 - For the Linux flatpak you must prepend commands with ``flatpak-spawn --host``.
 
+HDR Support
+-----------
+Streaming HDR content is supported for Windows hosts with NVIDIA, AMD, or Intel GPUs that support encoding HEVC Main 10.
+You must have an HDR-capable display or EDID emulator dongle connected to your host PC to activate HDR in Windows.
+
+- Ensure you enable the HDR option in your Moonlight client settings, otherwise the stream will be SDR.
+- A good HDR experience relies on proper HDR display calibration both in Windows and in game. HDR calibration can differ significantly between client and host displays.
+- We recommend calibrating the display by streaming the Windows HDR Calibration app to your client device and saving an HDR calibration profile to use while streaming.
+- You may also need to tune the brightness slider or HDR calibration options in game to the different HDR brightness capabilities of your client's display.
+- Older games that use NVIDIA-specific NVAPI HDR rather than native Windows 10 OS HDR support may not display in HDR.
+- Some GPUs can produce lower image quality or encoding performance when streaming in HDR compared to SDR.
+
 Tutorials
 ---------
 Tutorial videos are available `here <https://www.youtube.com/playlist?list=PLMYr5_xSeuXAbhxYHz86hA1eCDugoxXY0>`_.

src/platform/windows/display.h

@@ -151,6 +151,7 @@ class display_base_t : public display_t {
   }
 
   const char *dxgi_format_to_string(DXGI_FORMAT format);
+  const char *colorspace_to_string(DXGI_COLOR_SPACE_TYPE type);
 
   virtual capture_e snapshot(img_t *img, std::chrono::milliseconds timeout, bool cursor_visible) = 0;
   virtual int complete_img(img_t *img, bool dummy)                                               = 0;

src/platform/windows/display_base.cpp

@@ -563,6 +563,25 @@ int display_base_t::init(const ::video::config_t &config, const std::string &dis
   BOOST_LOG(info) << "Desktop resolution ["sv << dup_desc.ModeDesc.Width << 'x' << dup_desc.ModeDesc.Height << ']';
   BOOST_LOG(info) << "Desktop format ["sv << dxgi_format_to_string(dup_desc.ModeDesc.Format) << ']';
 
+  dxgi::output6_t output6 {};
+  status = output->QueryInterface(IID_IDXGIOutput6, (void **)&output6);
+  if(SUCCEEDED(status)) {
+    DXGI_OUTPUT_DESC1 desc1;
+    output6->GetDesc1(&desc1);
+
+    BOOST_LOG(info)
+      << std::endl
+      << "Colorspace         : "sv << colorspace_to_string(desc1.ColorSpace) << std::endl
+      << "Bits Per Color     : "sv << desc1.BitsPerColor << std::endl
+      << "Red Primary        : ["sv << desc1.RedPrimary[0] << ',' << desc1.RedPrimary[1] << ']' << std::endl
+      << "Green Primary      : ["sv << desc1.GreenPrimary[0] << ',' << desc1.GreenPrimary[1] << ']' << std::endl
+      << "Blue Primary       : ["sv << desc1.BluePrimary[0] << ',' << desc1.BluePrimary[1] << ']' << std::endl
+      << "White Point        : ["sv << desc1.WhitePoint[0] << ',' << desc1.WhitePoint[1] << ']' << std::endl
+      << "Min Luminance      : "sv << desc1.MinLuminance << " nits"sv << std::endl
+      << "Max Luminance      : "sv << desc1.MaxLuminance << " nits"sv << std::endl
+      << "Max Full Luminance : "sv << desc1.MaxFullFrameLuminance << " nits"sv;
+  }
+
   // Capture format will be determined from the first call to AcquireNextFrame()
   capture_format = DXGI_FORMAT_UNKNOWN;
 
@@ -598,6 +617,23 @@ bool display_base_t::get_hdr_metadata(SS_HDR_METADATA &metadata) {
   DXGI_OUTPUT_DESC1 desc1;
   output6->GetDesc1(&desc1);
 
+  // The primaries reported here seem to correspond to scRGB (Rec. 709)
+  // which we then convert to Rec 2020 in our scRGB FP16 -> PQ shader
+  // prior to encoding. It's not clear to me if we're supposed to report
+  // the primaries of the original colorspace or the one we've converted
+  // it to, but let's just report Rec 2020 primaries and D65 white level
+  // to avoid confusing clients by reporting Rec 709 primaries with a
+  // Rec 2020 colorspace. It seems like most clients ignore the primaries
+  // in the metadata anyway (luminance range is most important).
+  desc1.RedPrimary[0]   = 0.708f;
+  desc1.RedPrimary[1]   = 0.292f;
+  desc1.GreenPrimary[0] = 0.170f;
+  desc1.GreenPrimary[1] = 0.797f;
+  desc1.BluePrimary[0]  = 0.131f;
+  desc1.BluePrimary[1]  = 0.046f;
+  desc1.WhitePoint[0]   = 0.3127f;
+  desc1.WhitePoint[1]   = 0.3290f;
+
   metadata.displayPrimaries[0].x = desc1.RedPrimary[0] * 50000;
   metadata.displayPrimaries[0].y = desc1.RedPrimary[1] * 50000;
   metadata.displayPrimaries[1].x = desc1.GreenPrimary[0] * 50000;
@@ -749,6 +785,43 @@ const char *display_base_t::dxgi_format_to_string(DXGI_FORMAT format) {
   return format_str[format];
 }
 
+const char *display_base_t::colorspace_to_string(DXGI_COLOR_SPACE_TYPE type) {
+  const char *type_str[] = {
+    "DXGI_COLOR_SPACE_RGB_FULL_G22_NONE_P709",
+    "DXGI_COLOR_SPACE_RGB_FULL_G10_NONE_P709",
+    "DXGI_COLOR_SPACE_RGB_STUDIO_G22_NONE_P709",
+    "DXGI_COLOR_SPACE_RGB_STUDIO_G22_NONE_P2020",
+    "DXGI_COLOR_SPACE_RESERVED",
+    "DXGI_COLOR_SPACE_YCBCR_FULL_G22_NONE_P709_X601",
+    "DXGI_COLOR_SPACE_YCBCR_STUDIO_G22_LEFT_P601",
+    "DXGI_COLOR_SPACE_YCBCR_FULL_G22_LEFT_P601",
+    "DXGI_COLOR_SPACE_YCBCR_STUDIO_G22_LEFT_P709",
+    "DXGI_COLOR_SPACE_YCBCR_FULL_G22_LEFT_P709",
+    "DXGI_COLOR_SPACE_YCBCR_STUDIO_G22_LEFT_P2020",
+    "DXGI_COLOR_SPACE_YCBCR_FULL_G22_LEFT_P2020",
+    "DXGI_COLOR_SPACE_RGB_FULL_G2084_NONE_P2020",
+    "DXGI_COLOR_SPACE_YCBCR_STUDIO_G2084_LEFT_P2020",
+    "DXGI_COLOR_SPACE_RGB_STUDIO_G2084_NONE_P2020",
+    "DXGI_COLOR_SPACE_YCBCR_STUDIO_G22_TOPLEFT_P2020",
+    "DXGI_COLOR_SPACE_YCBCR_STUDIO_G2084_TOPLEFT_P2020",
+    "DXGI_COLOR_SPACE_RGB_FULL_G22_NONE_P2020",
+    "DXGI_COLOR_SPACE_YCBCR_STUDIO_GHLG_TOPLEFT_P2020",
+    "DXGI_COLOR_SPACE_YCBCR_FULL_GHLG_TOPLEFT_P2020",
+    "DXGI_COLOR_SPACE_RGB_STUDIO_G24_NONE_P709",
+    "DXGI_COLOR_SPACE_RGB_STUDIO_G24_NONE_P2020",
+    "DXGI_COLOR_SPACE_YCBCR_STUDIO_G24_LEFT_P709",
+    "DXGI_COLOR_SPACE_YCBCR_STUDIO_G24_LEFT_P2020",
+    "DXGI_COLOR_SPACE_YCBCR_STUDIO_G24_TOPLEFT_P2020",
+  };
+
+  if(type < ARRAYSIZE(type_str)) {
+    return type_str[type];
+  }
+  else {
+    return "UNKNOWN";
+  }
+}
+
 } // namespace platf::dxgi
 
 namespace platf {

src/platform/windows/display_vram.cpp

@@ -89,9 +89,12 @@ blend_t make_blend(device_t::pointer device, bool enable, bool invert) {
 
 blob_t convert_UV_vs_hlsl;
 blob_t convert_UV_ps_hlsl;
+blob_t convert_UV_PQ_ps_hlsl;
 blob_t scene_vs_hlsl;
 blob_t convert_Y_ps_hlsl;
+blob_t convert_Y_PQ_ps_hlsl;
 blob_t scene_ps_hlsl;
+blob_t scene_NW_ps_hlsl;
 
 struct img_d3d_t : public platf::img_t {
   std::shared_ptr<platf::display_t> display;
@@ -546,28 +549,39 @@ class hwdevice_t : public platf::hwdevice_t {
       return -1;
     }
 
-    status = device->CreatePixelShader(convert_Y_ps_hlsl->GetBufferPointer(), convert_Y_ps_hlsl->GetBufferSize(), nullptr, &convert_Y_ps);
+    status = device->CreateVertexShader(convert_UV_vs_hlsl->GetBufferPointer(), convert_UV_vs_hlsl->GetBufferSize(), nullptr, &convert_UV_vs);
     if(status) {
-      BOOST_LOG(error) << "Failed to create convertY pixel shader [0x"sv << util::hex(status).to_string_view() << ']';
+      BOOST_LOG(error) << "Failed to create convertUV vertex shader [0x"sv << util::hex(status).to_string_view() << ']';
       return -1;
     }
 
-    status = device->CreatePixelShader(convert_UV_ps_hlsl->GetBufferPointer(), convert_UV_ps_hlsl->GetBufferSize(), nullptr, &convert_UV_ps);
-    if(status) {
-      BOOST_LOG(error) << "Failed to create convertUV pixel shader [0x"sv << util::hex(status).to_string_view() << ']';
-      return -1;
-    }
+    // If the display is in HDR and we're streaming HDR, we'll be converting scRGB to SMPTE 2084 PQ.
+    // NB: We can consume scRGB in SDR with our regular shaders because it behaves like UNORM input.
+    if(format == DXGI_FORMAT_P010 && display->is_hdr()) {
+      status = device->CreatePixelShader(convert_Y_PQ_ps_hlsl->GetBufferPointer(), convert_Y_PQ_ps_hlsl->GetBufferSize(), nullptr, &convert_Y_ps);
+      if(status) {
+        BOOST_LOG(error) << "Failed to create convertY pixel shader [0x"sv << util::hex(status).to_string_view() << ']';
+        return -1;
+      }
 
-    status = device->CreateVertexShader(convert_UV_vs_hlsl->GetBufferPointer(), convert_UV_vs_hlsl->GetBufferSize(), nullptr, &convert_UV_vs);
-    if(status) {
-      BOOST_LOG(error) << "Failed to create convertUV vertex shader [0x"sv << util::hex(status).to_string_view() << ']';
-      return -1;
+      status = device->CreatePixelShader(convert_UV_PQ_ps_hlsl->GetBufferPointer(), convert_UV_PQ_ps_hlsl->GetBufferSize(), nullptr, &convert_UV_ps);
+      if(status) {
+        BOOST_LOG(error) << "Failed to create convertUV pixel shader [0x"sv << util::hex(status).to_string_view() << ']';
+        return -1;
+      }
     }
+    else {
+      status = device->CreatePixelShader(convert_Y_ps_hlsl->GetBufferPointer(), convert_Y_ps_hlsl->GetBufferSize(), nullptr, &convert_Y_ps);
+      if(status) {
+        BOOST_LOG(error) << "Failed to create convertY pixel shader [0x"sv << util::hex(status).to_string_view() << ']';
+        return -1;
+      }
 
-    status = device->CreatePixelShader(scene_ps_hlsl->GetBufferPointer(), scene_ps_hlsl->GetBufferSize(), nullptr, &scene_ps);
-    if(status) {
-      BOOST_LOG(error) << "Failed to create scene pixel shader [0x"sv << util::hex(status).to_string_view() << ']';
-      return -1;
+      status = device->CreatePixelShader(convert_UV_ps_hlsl->GetBufferPointer(), convert_UV_ps_hlsl->GetBufferSize(), nullptr, &convert_UV_ps);
+      if(status) {
+        BOOST_LOG(error) << "Failed to create convertUV pixel shader [0x"sv << util::hex(status).to_string_view() << ']';
+        return -1;
+      }
     }
 
     color_matrix = make_buffer(device.get(), ::video::colors[0]);
@@ -708,7 +722,6 @@ class hwdevice_t : public platf::hwdevice_t {
   vs_t convert_UV_vs;
   ps_t convert_UV_ps;
   ps_t convert_Y_ps;
-  ps_t scene_ps;
   vs_t scene_vs;
 
   D3D11_VIEWPORT outY_view;
@@ -978,10 +991,32 @@ int display_vram_t::init(const ::video::config_t &config, const std::string &dis
     return -1;
   }
 
-  status = device->CreatePixelShader(scene_ps_hlsl->GetBufferPointer(), scene_ps_hlsl->GetBufferSize(), nullptr, &scene_ps);
-  if(status) {
-    BOOST_LOG(error) << "Failed to create scene pixel shader [0x"sv << util::hex(status).to_string_view() << ']';
-    return -1;
+  if(config.dynamicRange && is_hdr()) {
+    // This shader will normalize scRGB white levels to a user-defined white level
+    status = device->CreatePixelShader(scene_NW_ps_hlsl->GetBufferPointer(), scene_NW_ps_hlsl->GetBufferSize(), nullptr, &scene_ps);
+    if(status) {
+      BOOST_LOG(error) << "Failed to create scene pixel shader [0x"sv << util::hex(status).to_string_view() << ']';
+      return -1;
+    }
+
+    // Use a 300 nit target for the mouse cursor. We should really get
+    // the user's SDR white level in nits, but there is no API that
+    // provides that information to Win32 apps.
+    float sdr_multiplier_data[16 / sizeof(float)] { 300.0f / 80.f }; // aligned to 16-byte
+    auto sdr_multiplier = make_buffer(device.get(), sdr_multiplier_data);
+    if(!sdr_multiplier) {
+      BOOST_LOG(warning) << "Failed to create SDR multiplier"sv;
+      return -1;
+    }
+
+    device_ctx->PSSetConstantBuffers(0, 1, &sdr_multiplier);
+  }
+  else {
+    status = device->CreatePixelShader(scene_ps_hlsl->GetBufferPointer(), scene_ps_hlsl->GetBufferSize(), nullptr, &scene_ps);
+    if(status) {
+      BOOST_LOG(error) << "Failed to create scene pixel shader [0x"sv << util::hex(status).to_string_view() << ']';
+      return -1;
+    }
   }
 
   blend_alpha   = make_blend(device.get(), true, false);
@@ -1108,7 +1143,25 @@ std::vector<DXGI_FORMAT> display_vram_t::get_supported_sdr_capture_formats() {
 }
 
 std::vector<DXGI_FORMAT> display_vram_t::get_supported_hdr_capture_formats() {
-  return { DXGI_FORMAT_R10G10B10A2_UNORM };
+  return {
+    // scRGB FP16 is the desired format for HDR content. This will also handle
+    // 10-bit SDR displays with the increased precision of FP16 vs 8-bit UNORMs.
+    DXGI_FORMAT_R16G16B16A16_FLOAT,
+
+    // DXGI_FORMAT_R10G10B10A2_UNORM seems like it might give us frames already
+    // converted to SMPTE 2084 PQ, however it seems to actually just clamp the
+    // scRGB FP16 values that DWM is using when the desktop format is scRGB FP16.
+    //
+    // If there is a case where the desktop format is really SMPTE 2084 PQ, it
+    // might make sense to support capturing it without conversion to scRGB,
+    // but we avoid it for now.
+
+    // We include the 8-bit modes too for when the display is in SDR mode,
+    // while the client stream is HDR-capable. These UNORM formats behave
+    // like a degenerate case of scRGB FP16 with values between 0.0f-1.0f.
+    DXGI_FORMAT_B8G8R8A8_UNORM,
+    DXGI_FORMAT_R8G8B8A8_UNORM,
+  };
 }
 
 std::shared_ptr<platf::hwdevice_t> display_vram_t::make_hwdevice(pix_fmt_e pix_fmt) {
@@ -1144,11 +1197,21 @@ int init() {
     return -1;
   }
 
+  convert_Y_PQ_ps_hlsl = compile_pixel_shader(SUNSHINE_SHADERS_DIR "/ConvertYPS_PQ.hlsl");
+  if(!convert_Y_PQ_ps_hlsl) {
+    return -1;
+  }
+
   convert_UV_ps_hlsl = compile_pixel_shader(SUNSHINE_SHADERS_DIR "/ConvertUVPS.hlsl");
   if(!convert_UV_ps_hlsl) {
     return -1;
   }
 
+  convert_UV_PQ_ps_hlsl = compile_pixel_shader(SUNSHINE_SHADERS_DIR "/ConvertUVPS_PQ.hlsl");
+  if(!convert_UV_PQ_ps_hlsl) {
+    return -1;
+  }
+
   convert_UV_vs_hlsl = compile_vertex_shader(SUNSHINE_SHADERS_DIR "/ConvertUVVS.hlsl");
   if(!convert_UV_vs_hlsl) {
     return -1;
@@ -1158,6 +1221,11 @@ int init() {
   if(!scene_ps_hlsl) {
     return -1;
   }
+
+  scene_NW_ps_hlsl = compile_pixel_shader(SUNSHINE_SHADERS_DIR "/ScenePS_NW.hlsl");
+  if(!scene_NW_ps_hlsl) {
+    return -1;
+  }
   BOOST_LOG(info) << "Compiled shaders"sv;
 
   return 0;

src/stream.cpp

@@ -820,8 +820,11 @@ void controlBroadcastThread(control_server_t *server) {
           send_rumble(session, rumble->id, rumble->lowfreq, rumble->highfreq);
         }
 
+        // Unlike rumble which we send as best-effort, HDR state messages are critical
+        // for proper functioning of some clients. We must wait to pop entries from
+        // the queue until we're sure we have a peer to send them to.
         auto &hdr_queue = session->control.hdr_queue;
-        while(hdr_queue->peek()) {
+        while(session->control.peer && hdr_queue->peek()) {
           auto hdr_info = hdr_queue->pop();
 
           send_hdr_mode(session, std::move(hdr_info));

src/video.cpp

@@ -1002,35 +1002,46 @@ std::optional<session_t> make_session(platf::display_t *disp, const encoder_t &e
   ctx->color_range = (config.encoderCscMode & 0x1) ? AVCOL_RANGE_JPEG : AVCOL_RANGE_MPEG;
 
   int sws_color_space;
-  switch(config.encoderCscMode >> 1) {
-  case 0:
-  default:
-    // Rec. 601
-    BOOST_LOG(info) << "Color coding [Rec. 601]"sv;
-    ctx->color_primaries = AVCOL_PRI_SMPTE170M;
-    ctx->color_trc       = AVCOL_TRC_SMPTE170M;
-    ctx->colorspace      = AVCOL_SPC_SMPTE170M;
-    sws_color_space      = SWS_CS_SMPTE170M;
-    break;
-
-  case 1:
-    // Rec. 709
-    BOOST_LOG(info) << "Color coding [Rec. 709]"sv;
-    ctx->color_primaries = AVCOL_PRI_BT709;
-    ctx->color_trc       = AVCOL_TRC_BT709;
-    ctx->colorspace      = AVCOL_SPC_BT709;
-    sws_color_space      = SWS_CS_ITU709;
-    break;
-
-  case 2:
-    // Rec. 2020
-    BOOST_LOG(info) << "Color coding [Rec. 2020]"sv;
+  if(config.dynamicRange && disp->is_hdr()) {
+    // When HDR is active, that overrides the colorspace the client requested
+    BOOST_LOG(info) << "HDR color coding [Rec. 2020 + SMPTE 2084 PQ]"sv;
     ctx->color_primaries = AVCOL_PRI_BT2020;
-    ctx->color_trc       = AVCOL_TRC_BT2020_10;
+    ctx->color_trc       = AVCOL_TRC_SMPTE2084;
     ctx->colorspace      = AVCOL_SPC_BT2020_NCL;
     sws_color_space      = SWS_CS_BT2020;
-    break;
   }
+  else {
+    switch(config.encoderCscMode >> 1) {
+    case 0:
+    default:
+      // Rec. 601
+      BOOST_LOG(info) << "SDR color coding [Rec. 601]"sv;
+      ctx->color_primaries = AVCOL_PRI_SMPTE170M;
+      ctx->color_trc       = AVCOL_TRC_SMPTE170M;
+      ctx->colorspace      = AVCOL_SPC_SMPTE170M;
+      sws_color_space      = SWS_CS_SMPTE170M;
+      break;
+
+    case 1:
+      // Rec. 709
+      BOOST_LOG(info) << "SDR color coding [Rec. 709]"sv;
+      ctx->color_primaries = AVCOL_PRI_BT709;
+      ctx->color_trc       = AVCOL_TRC_BT709;
+      ctx->colorspace      = AVCOL_SPC_BT709;
+      sws_color_space      = SWS_CS_ITU709;
+      break;
+
+    case 2:
+      // Rec. 2020
+      BOOST_LOG(info) << "SDR color coding [Rec. 2020]"sv;
+      ctx->color_primaries = AVCOL_PRI_BT2020;
+      ctx->color_trc       = AVCOL_TRC_BT2020_10;
+      ctx->colorspace      = AVCOL_SPC_BT2020_NCL;
+      sws_color_space      = SWS_CS_BT2020;
+      break;
+    }
+  }
+
   BOOST_LOG(info) << "Color range: ["sv << ((config.encoderCscMode & 0x1) ? "JPEG"sv : "MPEG"sv) << ']';
 
   AVPixelFormat sw_fmt;
@@ -1039,15 +1050,6 @@ std::optional<session_t> make_session(platf::display_t *disp, const encoder_t &e
   }
   else {
     sw_fmt = encoder.dynamic_pix_fmt;
-
-    // When HDR is active, that overrides the colorspace the client requested
-    if(disp->is_hdr()) {
-      BOOST_LOG(info) << "HDR color coding override [SMPTE ST 2084 PQ]"sv;
-      ctx->color_primaries = AVCOL_PRI_BT2020;
-      ctx->color_trc       = AVCOL_TRC_SMPTE2084;
-      ctx->colorspace      = AVCOL_SPC_BT2020_NCL;
-      sws_color_space      = SWS_CS_BT2020;
-    }
   }
 
   // Used by cbs::make_sps_hevc