From e39344fe8ff411b218945c614a84a73c574f0ef6 Mon Sep 17 00:00:00 2001 From: Andy Grundman Date: Sat, 13 Sep 2025 20:11:01 -0400 Subject: [PATCH 01/22] Bump vcpkg to 2028.08.27 and pin ffmpeg to 7.1.1 --- vcpkg | 2 +- vcpkg.json | 44 ++++++++++++++++++++++++++++---------------- 2 files changed, 29 insertions(+), 17 deletions(-) diff --git a/vcpkg b/vcpkg index 5fa0f075..120deac3 160000 --- a/vcpkg +++ b/vcpkg @@ -1 +1 @@ -Subproject commit 5fa0f075ea51f305b627ecd5e050a363707353ff +Subproject commit 120deac3062162151622ca4860575a33844ba10b diff --git a/vcpkg.json b/vcpkg.json index 643c72a9..a9c55497 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -1,19 +1,31 @@ { - "name": "moonlight-xbox", + "name": "moonlight-xbox", + "builtin-baseline": "120deac3062162151622ca4860575a33844ba10b", "dependencies": [ - { - "name": "ffmpeg", - "default-features": false, - "features": ["avcodec","avdevice","swscale","x264"] - }, - "curl", - "openssl", - "expat", - "zlib", - "nlohmann-json", - "bzip2", - "freetype", - "opus", - "mdns" + { + "name": "ffmpeg", + "default-features": false, + "features": [ + "avcodec", + "avdevice", + "swscale", + "x264" + ] + }, + "curl", + "openssl", + "expat", + "zlib", + "nlohmann-json", + "bzip2", + "freetype", + "opus", + "mdns" + ], + "overrides": [ + { + "name": "ffmpeg", + "version": "7.1.1" + } ] -} \ No newline at end of file +} From 650491e79fbab8c11f8dbf60fdcbc2b15bd10365 Mon Sep 17 00:00:00 2001 From: Andy Grundman Date: Tue, 9 Sep 2025 12:07:06 -0400 Subject: [PATCH 02/22] Port moonlight-qt's Pacer class, with a very basic integration. --- Common/DeviceResources.cpp | 24 +- Common/DeviceResources.h | 2 + Pages/StreamPage.xaml.cpp | 10 +- Plot/ImGuiPlots.cpp | 2 + Plot/PlotDesc.h | 14 +- State/MoonlightClient.cpp | 4 +- State/Stats.cpp | 52 ++--- State/Stats.h | 3 +- Streaming/FFmpegDecoder.cpp | 325 ++++++++++++-------------- Streaming/FFmpegDecoder.h | 34 +-- Streaming/Pacer.cpp | 174 ++++++++++++++ Streaming/Pacer.h | 42 ++++ Streaming/StatsRenderer.cpp | 13 +- Streaming/VideoRenderer.cpp | 82 +++---- Streaming/VideoRenderer.h | 6 +- Streaming/moonlight_xbox_dxMain.cpp | 14 +- Streaming/moonlight_xbox_dxMain.h | 2 +- moonlight-xbox-dx.vcxproj | 3 + moonlight-xbox-dx.vcxproj.filters | 342 ++++++++++++++++++++++++++++ pch.h | 16 ++ 20 files changed, 858 insertions(+), 306 deletions(-) create mode 100644 Streaming/Pacer.cpp create mode 100644 Streaming/Pacer.h diff --git a/Common/DeviceResources.cpp b/Common/DeviceResources.cpp index a3c10e6c..4d8e5a97 100644 --- a/Common/DeviceResources.cpp +++ b/Common/DeviceResources.cpp @@ -63,6 +63,7 @@ DX::DeviceResources::DeviceResources() : m_d3dRenderTargetSize(), m_dxgiFactoryFlags(0), m_enableVsync(true), + m_frameLatencyWaitableObject(), m_swapchainVsync(true), m_forceTearing(false), m_outputSize(), @@ -249,7 +250,7 @@ void DX::DeviceResources::CreateWindowSizeDependentResources() lround(m_d3dRenderTargetSize.Width), lround(m_d3dRenderTargetSize.Height), m_backBufferFormat, - (m_enableVsync && !m_forceTearing) ? 0 : DXGI_SWAP_CHAIN_FLAG_ALLOW_TEARING + 0 ); Utils::Logf("ResizeBuffers() ALLOW_TEARING=%d\n", (m_enableVsync && !m_forceTearing) ? 0 : 1); @@ -283,7 +284,7 @@ void DX::DeviceResources::CreateWindowSizeDependentResources() //Check moonlight-stream/moonlight-qt/app/streaming/video/ffmpeg-renderers/d3d11va.cpp for rationale swapChainDesc.BufferCount = 5; swapChainDesc.SwapEffect = DXGI_SWAP_EFFECT_FLIP_DISCARD; - swapChainDesc.Flags = (m_enableVsync && !m_forceTearing) ? 0 : DXGI_SWAP_CHAIN_FLAG_ALLOW_TEARING; + swapChainDesc.Flags = 0; swapChainDesc.Scaling = DXGI_SCALING_STRETCH; swapChainDesc.AlphaMode = DXGI_ALPHA_MODE_IGNORE; @@ -334,9 +335,9 @@ void DX::DeviceResources::CreateWindowSizeDependentResources() swapChain.As(&m_swapChain) ); - // We must not call this for flip swapchains. It will counterintuitively - // increase latency by forcing our Present() to block on DWM even when - // DX::ThrowIfFailed(dxgiDevice->SetMaximumFrameLatency(1)); + // This controls how GetFrameLatencyWaitableObject operates + //DX::ThrowIfFailed(m_swapChain->SetMaximumFrameLatency(1)); + //m_frameLatencyWaitableObject = m_swapChain->GetFrameLatencyWaitableObject(); // Associate swap chain with SwapChainPanel // UI changes will need to be dispatched back to the UI thread @@ -693,11 +694,6 @@ void DX::DeviceResources::Present() else { DX::ThrowIfFailed(hr); } - - auto ffmpeg = moonlight_xbox_dx::FFMpegDecoder::getInstance(); - if (ffmpeg != nullptr && ffmpeg->shouldUnlock) { - ffmpeg->mutex.unlock(); - } } // This method determines the rotation between the display device's native orientation and the @@ -755,6 +751,14 @@ DXGI_MODE_ROTATION DX::DeviceResources::ComputeDisplayRotation() return rotation; } +// Block the current thread until the swap chain has finished presenting. +void DX::DeviceResources::WaitOnSwapChain() +{ + DWORD result = WaitForSingleObjectEx( + m_frameLatencyWaitableObject, + 1000, // 1 second timeout (shouldn't ever occur) + true); +} //Thank you tunip3 for https://github.com/libretro/RetroArch/pull/13406/files //Check if we are running on Xbox diff --git a/Common/DeviceResources.h b/Common/DeviceResources.h index 1734b962..686cdfd7 100644 --- a/Common/DeviceResources.h +++ b/Common/DeviceResources.h @@ -27,6 +27,7 @@ namespace DX void RegisterDeviceNotify(IDeviceNotify* deviceNotify); void Trim(); void Present(); + void WaitOnSwapChain(); void GetUWPPixelDimensions(uint32_t *width, uint32_t *height); double GetUWPRefreshRate(); void SetForceTearing(bool forceTearing); @@ -85,6 +86,7 @@ namespace DX Microsoft::WRL::ComPtr m_d3dDevice; Microsoft::WRL::ComPtr m_d3dContext; Microsoft::WRL::ComPtr m_swapChain; + HANDLE m_frameLatencyWaitableObject; // Direct3D rendering objects. Required for 3D. Microsoft::WRL::ComPtr m_d3dRenderTargetView; diff --git a/Pages/StreamPage.xaml.cpp b/Pages/StreamPage.xaml.cpp index eeaec5cc..79e0ebff 100644 --- a/Pages/StreamPage.xaml.cpp +++ b/Pages/StreamPage.xaml.cpp @@ -233,20 +233,14 @@ void StreamPage::toggleHDR_WinAltB_Click(Platform::Object^ sender, Windows::UI:: void StreamPage::increaseFrameDropTarget_Click(Platform::Object^ sender, Windows::UI::Xaml::RoutedEventArgs^ e) { int target = 2; - FFMpegDecoder* ffmpeg = FFMpegDecoder::getInstance(); - if (ffmpeg) { - target = ffmpeg->ModifyFrameDropTarget(true); // true increases the value - } + target = FFMpegDecoder::instance().ModifyFrameDropTarget(true); // true increases the value this->frameDropTargetLabel->Text = Utils::StringPrintf("Frame queue size: %d", target); } void StreamPage::decreaseFrameDropTarget_Click(Platform::Object^ sender, Windows::UI::Xaml::RoutedEventArgs^ e) { int target = 2; - FFMpegDecoder* ffmpeg = FFMpegDecoder::getInstance(); - if (ffmpeg) { - target = ffmpeg->ModifyFrameDropTarget(false); // false decreases the value - } + target = FFMpegDecoder::instance().ModifyFrameDropTarget(false); // false decreases the value this->frameDropTargetLabel->Text = Utils::StringPrintf("Frame queue size: %d", target); } diff --git a/Plot/ImGuiPlots.cpp b/Plot/ImGuiPlots.cpp index 08aa9488..7348d603 100644 --- a/Plot/ImGuiPlots.cpp +++ b/Plot/ImGuiPlots.cpp @@ -25,6 +25,8 @@ ImGuiPlots::ImGuiPlots() : Plot(kPlotDescs[PLOT_DROPPED_NETWORK]), Plot(kPlotDescs[PLOT_DROPPED_PACER]), Plot(kPlotDescs[PLOT_OVERHEAD]), + + Plot(kPlotDescs[PLOT_ETC]), }} { } diff --git a/Plot/PlotDesc.h b/Plot/PlotDesc.h index 4a1b8b73..313707cf 100644 --- a/Plot/PlotDesc.h +++ b/Plot/PlotDesc.h @@ -11,6 +11,7 @@ enum PlotType PLOT_DROPPED_NETWORK, PLOT_DROPPED_PACER, PLOT_OVERHEAD, + PLOT_ETC, PlotCount }; @@ -34,10 +35,11 @@ struct PlotDesc // clang-format off inline constexpr std::array kPlotDescs = {{ - {"Frametime", PLOT_LABEL_MIN_MAX_AVG, "ms", -0.1, 50.0, 0.0, 49.9}, - {"Host Frametime", PLOT_LABEL_MIN_MAX_AVG, "ms", -0.1, 50.0, 0.0, 49.9}, - {"Queued Frames", PLOT_LABEL_MIN_MAX_AVG, "", -1.0, 10.0, 0.0, 0.0}, - {"Dropped Frames (network)", PLOT_LABEL_TOTAL_INT, "", -1.0, 3.0, 0.0, 0.0}, - {"Dropped Frames (pacing queue)", PLOT_LABEL_TOTAL_INT, "", -1.0, 3.0, 0.0, 0.0}, - {"Graph overhead", PLOT_LABEL_MIN_MAX_AVG, "ms", -0.1, 6.0, 0.0, 0.0}, + {"Frametime", PLOT_LABEL_MIN_MAX_AVG, "ms", -0.1f, 50.0f, 0.0f, 49.9f}, + {"Host Frametime", PLOT_LABEL_MIN_MAX_AVG, "ms", -0.1f, 50.0f, 0.0f, 49.9f}, + {"Queued Frames", PLOT_LABEL_MIN_MAX_AVG, "", -1.0f, 10.0f, 0.0f, 0.0f}, + {"Dropped Frames (network)", PLOT_LABEL_TOTAL_INT, "", -1.0f, 3.0f, 0.0f, 0.0f}, + {"Dropped Frames (pacing queue)", PLOT_LABEL_TOTAL_INT, "", -1.0f, 3.0f, 0.0f, 0.0f}, + {"Graph overhead", PLOT_LABEL_MIN_MAX_AVG, "ms", -0.1f, 6.0f, 0.0f, 0.0f}, + {"Etc...", PLOT_LABEL_MIN_MAX_AVG, "ms", -0.1f, 50.0f, 0.0f, 49.9f}, }}; diff --git a/State/MoonlightClient.cpp b/State/MoonlightClient.cpp index 08a65148..6b504920 100644 --- a/State/MoonlightClient.cpp +++ b/State/MoonlightClient.cpp @@ -271,8 +271,10 @@ int MoonlightClient::StartStreaming(std::shared_ptr res, St callbacks.setHdrMode = connection_set_hdr; callbacks.rumble = connection_rumble; //callbacks.rumbleTriggers = connection_trigger_rumble; - FFMpegDecoder::createDecoderInstance(res); + + FFMpegDecoder::instance().CompleteInitialization(res, &config); DECODER_RENDERER_CALLBACKS rCallbacks = FFMpegDecoder::getDecoder(); + AUDIO_RENDERER_CALLBACKS aCallbacks = AudioPlayer::getDecoder(); int k = LiStartConnection(&serverData.serverInfo, &config, &callbacks, &rCallbacks, &aCallbacks, NULL, 0, NULL, 0); sprintf(message, "LiStartConnection %d\n", k); diff --git a/State/Stats.cpp b/State/Stats.cpp index 82547866..a6c101fb 100644 --- a/State/Stats.cpp +++ b/State/Stats.cpp @@ -58,7 +58,8 @@ bool Stats::ShouldUpdateDisplay(DX::StepTimer const& timer, bool isVisible, char // Includes time spent in FEC reassembly // 3. Host processing latency (encode time) // 4. network packet loss (caller reports frame sequence number holes) -void Stats::SubmitVideoBytesAndReassemblyTime(uint32_t length, PDECODE_UNIT decodeUnit, uint32_t droppedFrames) { +void Stats::SubmitVideoBytesAndReassemblyTime(uint32_t length, PDECODE_UNIT decodeUnit, uint32_t droppedFrames) +{ std::lock_guard lock(m_mutex); m_ActiveWndVideoStats.receivedFrames++; m_ActiveWndVideoStats.totalFrames++; @@ -75,8 +76,7 @@ void Stats::SubmitVideoBytesAndReassemblyTime(uint32_t length, PDECODE_UNIT deco if (frameHPL != 0) { if (m_ActiveWndVideoStats.minHostProcessingLatency != 0) { m_ActiveWndVideoStats.minHostProcessingLatency = std::min(m_ActiveWndVideoStats.minHostProcessingLatency, frameHPL); - } - else { + } else { m_ActiveWndVideoStats.minHostProcessingLatency = frameHPL; } m_ActiveWndVideoStats.framesWithHostProcessingLatency += 1; @@ -99,7 +99,7 @@ void Stats::SubmitVideoBytesAndReassemblyTime(uint32_t length, PDECODE_UNIT deco lastHostPts = decodeUnit->presentationTimeMs; // Queued frames for graph - ImGuiPlots::instance().observeFloat(PLOT_QUEUED_FRAMES, LiGetPendingVideoFrames()); + //ImGuiPlots::instance().observeFloat(PLOT_QUEUED_FRAMES, LiGetPendingVideoFrames()); } // Time in milliseconds we spent decoding one frame, it is added up to later be divided by decodedFrames @@ -115,10 +115,13 @@ void Stats::SubmitDroppedFrame(int count) { m_ActiveWndVideoStats.pacerDroppedFrames += count; } -// Time in microseconds for all rendering, including Update(), Render(), and Present(). +// Time in microseconds we spent in the frame pacer, and time for rendering the frame. // Also increments the rendered frame count. -void Stats::SubmitRenderTime(uint64_t renderTimeQpc) { +void Stats::SubmitPacerTime(uint64_t pacerTimeQpc, uint64_t renderTimeQpc) { std::lock_guard lock(m_mutex); + uint64_t pacerTimeUs = QpcToUs(pacerTimeQpc); + m_ActiveWndVideoStats.totalPacerTimeUs += pacerTimeUs; + uint64_t renderTimeUs = QpcToUs(renderTimeQpc); m_ActiveWndVideoStats.totalRenderTimeUs += renderTimeUs; m_ActiveWndVideoStats.renderedFrames++; @@ -135,6 +138,7 @@ void Stats::addVideoStats(DX::StepTimer const& timer, VIDEO_STATS& src, VIDEO_ST dst.pacerDroppedFrames += src.pacerDroppedFrames; dst.totalReassemblyTime += src.totalReassemblyTime; dst.totalDecodeTime += src.totalDecodeTime; + dst.totalPacerTimeUs += src.totalPacerTimeUs; dst.totalRenderTimeUs += src.totalRenderTimeUs; if (dst.minHostProcessingLatency == 0) { @@ -174,11 +178,7 @@ void Stats::addVideoStats(DX::StepTimer const& timer, VIDEO_STATS& src, VIDEO_ST } void Stats::formatVideoStats(DX::StepTimer const& timer, VIDEO_STATS& stats, char* output, size_t length) { - FFMpegDecoder* ffmpeg = FFMpegDecoder::getInstance(); - if (!ffmpeg) { - output[0] = 0; - return; - } + FFMpegDecoder& ffmpeg = FFMpegDecoder::instance(); int offset = 0; const char* codecString; @@ -187,7 +187,7 @@ void Stats::formatVideoStats(DX::StepTimer const& timer, VIDEO_STATS& stats, cha // Start with an empty string output[offset] = 0; - switch (ffmpeg->videoFormat) + switch (ffmpeg.videoFormat) { case VIDEO_FORMAT_H264: codecString = "H.264"; @@ -255,22 +255,20 @@ void Stats::formatVideoStats(DX::StepTimer const& timer, VIDEO_STATS& stats, cha } if (stats.receivedFps > 0) { - if (ffmpeg != nullptr) { - ret = snprintf(&output[offset], - length - offset, - "Video stream: %dx%d %.2f FPS (Codec: %s)\n", - ffmpeg->width, - ffmpeg->height, - stats.totalFps, - codecString); - if (ret < 0 || ret >= length - offset) { - Utils::Log("Error: stringifyVideoStats length overflow\n"); - return; - } - - offset += ret; + ret = snprintf(&output[offset], + length - offset, + "Video stream: %dx%d %.2f FPS (Codec: %s)\n", + ffmpeg.width, + ffmpeg.height, + stats.totalFps, + codecString); + if (ret < 0 || ret >= length - offset) { + Utils::Log("Error: stringifyVideoStats length overflow\n"); + return; } + offset += ret; + double avgVideoMbps = m_bwTracker.GetAverageMbps(); double peakVideoMbps = m_bwTracker.GetPeakMbps(); @@ -337,12 +335,14 @@ void Stats::formatVideoStats(DX::StepTimer const& timer, VIDEO_STATS& stats, cha "Frames dropped due to network jitter: %.2f%%\n" "Average network latency: %s\n" "Average reassembly/decoding time: %.2f/%.2f ms\n" + "Average frame queue delay: %.2f ms\n" "Average render time: %.2f ms\n", stats.totalFrames ? (double)stats.networkDroppedFrames / stats.totalFrames * 100 : 0.0f, stats.totalFrames ? (double)stats.pacerDroppedFrames / stats.totalFrames * 100 : 0.0f, rttString, stats.decodedFrames ? (double)stats.totalReassemblyTime / stats.decodedFrames : 0.0f, stats.decodedFrames ? (double)stats.totalDecodeTime / stats.decodedFrames : 0.0f, + stats.renderedFrames ? (double)stats.totalPacerTimeUs / 1000.0 / stats.renderedFrames : 0.0f, stats.renderedFrames ? (double)stats.totalRenderTimeUs / 1000.0 / stats.renderedFrames : 0.0f); if (ret < 0 || ret >= length - offset) { Utils::Log("Error: stringifyVideoStats length overflow\n"); diff --git a/State/Stats.h b/State/Stats.h index cd73568a..fabfca33 100644 --- a/State/Stats.h +++ b/State/Stats.h @@ -32,6 +32,7 @@ typedef struct _VIDEO_STATS { uint32_t framesWithHostProcessingLatency; uint32_t totalReassemblyTime; double totalDecodeTime; + uint64_t totalPacerTimeUs; uint64_t totalRenderTimeUs; uint32_t lastRtt; uint32_t lastRttVariance; @@ -54,7 +55,7 @@ namespace moonlight_xbox_dx void SubmitVideoBytesAndReassemblyTime(uint32_t length, PDECODE_UNIT decodeUnit, uint32_t droppedFrames); void SubmitDecodeMs(double decodeMs); void SubmitDroppedFrame(int count); - void SubmitRenderTime(uint64_t renderTimeQpc); + void SubmitPacerTime(uint64_t pacerTimeQpc, uint64_t renderTimeQpc); void SetDisplayStatus(SyncMode status) { m_displayStatus = status; } diff --git a/Streaming/FFmpegDecoder.cpp b/Streaming/FFmpegDecoder.cpp index 6ffb866c..4266c0bf 100644 --- a/Streaming/FFmpegDecoder.cpp +++ b/Streaming/FFmpegDecoder.cpp @@ -17,9 +17,49 @@ extern "C" { #include #include } -#define DECODER_BUFFER_SIZE 1048576 + +using namespace moonlight_xbox_dx; + +#define INITIAL_DECODER_BUFFER_SIZE (256 * 1024) + +static bool ensure_buf_size(unsigned char **buf, int *buf_size, int required_size) +{ + if (*buf_size >= required_size) + return true; + + Utils::Logf("ensure_buf_size grew from %d -> %d\n", *buf_size, required_size); + + *buf_size = required_size; + *buf = (unsigned char *)realloc(*buf, *buf_size); + if (!*buf) { + return false; + } + + return true; +} namespace moonlight_xbox_dx { + FFMpegDecoder &FFMpegDecoder::instance() { + static FFMpegDecoder inst; + return inst; + } + + FFMpegDecoder::FFMpegDecoder(): + width(0), + height(0), + videoFormat(0), + decoder(nullptr), + decoder_ctx(nullptr), + device_ctx(nullptr), + d3d11va_device_ctx(nullptr), + ffmpeg_buffer(nullptr), + ffmpeg_buffer_size(0), + resources(nullptr), + decodeStartTime(0), + frameDropTarget(2), + m_LastFrameNumber(0), + m_Pacer(nullptr) { + } void lock_context(void* dec) { auto ff = (FFMpegDecoder*)dec; @@ -31,38 +71,28 @@ namespace moonlight_xbox_dx { ff->mutex.unlock(); } - enum AVPixelFormat ffGetFormat(AVCodecContext* context, const enum AVPixelFormat* pixFmts) - { - FFMpegDecoder* d = ((FFMpegDecoder*)context->opaque); - const enum AVPixelFormat* p; - - for (p = pixFmts; *p != -1; p++) { - // Only match our hardware decoding codec or preferred SW pixel - // format (if not using hardware decoding). It's crucial - // to override the default get_format() which will try - // to gracefully fall back to software decode and break us. - if (*p == AV_PIX_FMT_D3D11) { - return *p; - } - } + void ffmpeg_log_callback(void *ptr, int level, const char *fmt, va_list vl) { + char lineBuffer[1024]; + static int printPrefix = 1; + if ((level & 0xFF) > av_log_get_level()) { + return; + } + // We need to use the *previous* printPrefix value to determine whether to + // print the prefix this time. av_log_format_line() will set the printPrefix + // value to indicate whether the prefix should be printed *next time*. + bool shouldPrefixThisMessage = printPrefix != 0; - return AV_PIX_FMT_NONE; + av_log_format_line(ptr, level, fmt, vl, lineBuffer, sizeof(lineBuffer), &printPrefix); + Utils::Logf(shouldPrefixThisMessage ? "[ffmpeg] %s" : "%s", lineBuffer); } - void ffmpeg_log_callback(void* avcl, int level, const char* fmt, va_list vl) { - if (level > AV_LOG_INFO) return; - - char message[2048]; - vsprintf_s(message, fmt, vl); - OutputDebugStringA("[FFMPEG]"); - OutputDebugStringA(message); - if (level <= AV_LOG_INFO) { - Utils::Log("[FFMPEG]"); - Utils::Log(message); - Utils::Log("\n"); - } + void FFMpegDecoder::CompleteInitialization(const std::shared_ptr& res, STREAM_CONFIGURATION *config) { + this->resources = res; + + m_Pacer = std::make_unique(res); + m_Pacer->initialize(config->fps, res->GetRefreshRate()); } int FFMpegDecoder::Init(int videoFormat, int width, int height, int redrawRate, void* context, int drFlags) { @@ -72,15 +102,17 @@ namespace moonlight_xbox_dx { this->frameDropTarget = 2; // user can modify this value this->m_LastFrameNumber = 0; + this->ffmpeg_buffer_size = 0; #if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58,10,100) avcodec_register_all(); #endif - av_log_set_level(AV_LOG_INFO); + // Increase log level until the first frame is decoded + av_log_set_level(AV_LOG_DEBUG); + av_log_set_callback(&ffmpeg_log_callback); #pragma warning(suppress : 4996) - av_init_packet(&pkt); if (videoFormat & VIDEO_FORMAT_MASK_H264) { decoder = avcodec_find_decoder(AV_CODEC_ID_H264); @@ -125,7 +157,6 @@ namespace moonlight_xbox_dx { decoder_ctx->sw_pix_fmt = (videoFormat & VIDEO_FORMAT_MASK_10BIT) ? AV_PIX_FMT_P010 : AV_PIX_FMT_NV12; decoder_ctx->width = width; decoder_ctx->height = height; - decoder_ctx->get_format = ffGetFormat; int err = avcodec_open2(decoder_ctx, decoder, NULL); if (err < 0) { @@ -134,33 +165,17 @@ namespace moonlight_xbox_dx { Utils::Log(msg); return err; } - ffmpeg_buffer = (unsigned char*)malloc(DECODER_BUFFER_SIZE + AV_INPUT_BUFFER_PADDING_SIZE); - if (ffmpeg_buffer == NULL) { - Utils::Log("OOM\n"); - Cleanup(); - return -1; + + if (decoder_ctx->pix_fmt != AV_PIX_FMT_D3D11) { + Utils::Log("Warning: decoder did not select AV_PIX_FMT_D3D11\n"); } - int buffer_count = 2; - dec_frames_cnt = buffer_count; - dec_frames = (AVFrame**)malloc(buffer_count * sizeof(AVFrame*)); - ready_frames = (AVFrame**)malloc(buffer_count * sizeof(AVFrame*)); - if (dec_frames == NULL) { - Utils::Log("Cannot allocate Frames\n"); + + if (!ensure_buf_size(&ffmpeg_buffer, &ffmpeg_buffer_size, INITIAL_DECODER_BUFFER_SIZE + AV_INPUT_BUFFER_PADDING_SIZE)) { + Utils::Log("Couldn't allocate initial ffmpeg_buffer\n"); + Cleanup(); return -1; } - for (int i = 0; i < buffer_count; i++) { - dec_frames[i] = av_frame_alloc(); - if (dec_frames[i] == NULL) { - Utils::Log("Cannot allocate Frames\n"); - return -1; - } - ready_frames[i] = av_frame_alloc(); - if (ready_frames[i] == NULL) { - Utils::Log("Cannot allocate Frames\n"); - return -1; - } - } GAMING_DEVICE_MODEL_INFORMATION info = {}; GetGamingDeviceModelInformation(&info); if (info.vendorId == GAMING_DEVICE_VENDOR_ID_MICROSOFT && @@ -174,54 +189,37 @@ namespace moonlight_xbox_dx { return 0; } - void FFMpegDecoder::Start() { - Utils::Log("Decoding Started\n"); - } - - void FFMpegDecoder::Stop() { - Utils::Log("Decoding Stopped\n"); - } - void FFMpegDecoder::Cleanup() { - if (dec_frames != NULL) { - for (int i = 0; i < dec_frames_cnt; i++) { - av_frame_free(&dec_frames[i]); - } - free(dec_frames); - } - avcodec_close(decoder_ctx); avcodec_free_context(&decoder_ctx); if (ffmpeg_buffer != NULL) { free(ffmpeg_buffer); ffmpeg_buffer = NULL; - } - if (ready_frames != NULL) { - free(ready_frames); - ready_frames = NULL; + ffmpeg_buffer_size = 0; } m_LastFrameNumber = 0; - shouldUnlock = false; - Utils::Log("Decoding Clean\n"); + m_Pacer.reset(nullptr); + + Utils::Log("FFMpegDecoder::Cleanup\n"); } - bool FFMpegDecoder::SubmitDU() { - PDECODE_UNIT decodeUnit = nullptr; - VIDEO_FRAME_HANDLE frameHandle = nullptr; - bool status = LiWaitForNextVideoFrame(&frameHandle, &decodeUnit); - if (status == false)return false; - if (decodeUnit->fullLength > DECODER_BUFFER_SIZE) { - Utils::Log("(0) Decoder Buffer Size reached\n"); - LiCompleteVideoFrame(frameHandle, DR_NEED_IDR); - return false; - } + // Called by the VideoDec thread + int FFMpegDecoder::SubmitDecodeUnit(PDECODE_UNIT decodeUnit) { PLENTRY entry = decodeUnit->bufferList; - uint32_t length = 0; - while (entry != NULL) { - memcpy(ffmpeg_buffer + length, entry->data, entry->length); - length += entry->length; - entry = entry->next; + int length = 0; + int64_t decodeStart = av_gettime_relative(); + + if (!ensure_buf_size(&ffmpeg_buffer, &ffmpeg_buffer_size, decodeUnit->fullLength + AV_INPUT_BUFFER_PADDING_SIZE)) { + Utils::Logf("Couldn't realloc ffmpeg_buffer\n"); + return DR_NEED_IDR; } + while (entry != NULL) { + memcpy(ffmpeg_buffer + length, entry->data, entry->length); + length += entry->length; + entry = entry->next; + } + memset(ffmpeg_buffer + length, 0, AV_INPUT_BUFFER_PADDING_SIZE); + // Detect breaks in the frame sequence indicating dropped packets uint32_t droppedFramesNetwork = 0; if (m_LastFrameNumber > 0 && decodeUnit->frameNumber > (m_LastFrameNumber + 1)) { @@ -233,69 +231,58 @@ namespace moonlight_xbox_dx { // track stats for a variety of things we can track at the same time this->resources->GetStats()->SubmitVideoBytesAndReassemblyTime(length, decodeUnit, droppedFramesNetwork); - int err; - err = Decode(ffmpeg_buffer, length); + // ffmpeg_decode + AVPacket *pkt = av_packet_alloc(); + pkt->data = ffmpeg_buffer; + pkt->size = length; + int err = avcodec_send_packet(decoder_ctx, pkt); + av_packet_unref(pkt); if (err < 0) { - LiCompleteVideoFrame(frameHandle, DR_NEED_IDR); - return false; + char ffmpegError[1024]; + av_strerror(err, ffmpegError, 1024); + Utils::Logf("avcodec_send_packet failed: %s\n", ffmpegError); + return DR_NEED_IDR; } - LiCompleteVideoFrame(frameHandle, DR_OK); - return true; - } - int FFMpegDecoder::Decode(unsigned char* indata, int inlen) { - int err; + for (;;) { + AVFrame* frame = av_frame_alloc(); + err = avcodec_receive_frame(decoder_ctx, frame); + if (err == AVERROR(EAGAIN)) { + av_frame_free(&frame); + break; // decoder needs more input before producing another frame + } + if (err == AVERROR_EOF) { + av_frame_free(&frame); + break; + } + if (err < 0) { + char ffmpegError[1024]; + av_strerror(err, ffmpegError, sizeof(ffmpegError)); + Utils::Logf("avcodec_receive_frame failed: %s\n", ffmpegError); + av_frame_free(&frame); + return DR_NEED_IDR; + } - pkt.data = indata; - pkt.size = inlen; - decodeStartTime = av_gettime_relative(); - err = avcodec_send_packet(decoder_ctx, &pkt); - if (err < 0) { + // Capture a frame timestamp to measuring pacing delay + LARGE_INTEGER(afterDecode); + QueryPerformanceCounter(&afterDecode); + frame->pkt_dts = afterDecode.QuadPart; - char errorstringnew[2048], ffmpegError[1024]; - av_strerror(err, ffmpegError, 1024); - sprintf(errorstringnew, "Error avcodec_send_packet: %s\n", ffmpegError); - Utils::Log(errorstringnew); - return err == 1 ? 0 : err; - } - return err < 0 ? err : 0; - } + // Store the presentation time + frame->pts = decodeUnit->presentationTimeMs; - AVFrame* FFMpegDecoder::GetFrame() { - int err = avcodec_receive_frame(decoder_ctx, dec_frames[next_frame]); - if (err != 0 && err != AVERROR(EAGAIN)) { - char errorstringnew[1024]; - sprintf(errorstringnew, "Error avcodec_receive_frame: %d\n", AVERROR(err)); - Utils::Log(errorstringnew); - return nullptr; + // Queue the frame for rendering (or render now if pacer is disabled) + m_Pacer->submitFrame(frame); } - if (err == 0) { - AVFrame *frame = dec_frames[next_frame]; - - this->resources->GetStats()->SubmitDecodeMs((double)(av_gettime_relative() - decodeStartTime) / 1000.0); - - // Frame pacing (to be improved) - int dropTarget = this->frameDropTarget; - bool is_idr = (frame->pict_type == AV_PICTURE_TYPE_I) && (frame->flags & AV_FRAME_FLAG_KEY); - int pending = LiGetPendingVideoFrames(); - bool shouldDrop = pending > dropTarget && !is_idr; - ImGuiPlots::instance().observeFloat(PLOT_DROPPED_PACER, shouldDrop ? 1 : 0); - if (shouldDrop) { - Utils::Logf("dropping frame because pending frame count %d > frame queue size %d\n", pending, dropTarget); - // Drop frame to reduce latency (unless it's an IDR frame) - this->resources->GetStats()->SubmitDroppedFrame(1); - av_frame_unref(frame); - return nullptr; - } - //Not the best way to handle this. BUT IT DOES FIX XBOX ONES!!!! - //Honestly this did take too much time of my life to care to make a better version - //If you want to fix this, have fun! (And hopefully you have Microsoft blessing/tools/support for that) - if (hackWait && LiGetPendingVideoFrames() < 2)moonlight_xbox_dx::usleep(12000); + this->resources->GetStats()->SubmitDecodeMs(double(av_gettime_relative() - decodeStart) / 1000.0); - return frame; + // Restore default log level after a successful decode + if (av_log_get_level() > 0) { + av_log_set_level(AV_LOG_QUIET); } - return nullptr; + + return DR_OK; } int FFMpegDecoder::ModifyFrameDropTarget(bool increase) { @@ -316,51 +303,41 @@ namespace moonlight_xbox_dx { } //Helpers - FFMpegDecoder* instance; - - FFMpegDecoder* FFMpegDecoder::createDecoderInstance(std::shared_ptr resources) { - if (instance == NULL) { - instance = new FFMpegDecoder(resources); - } - return instance; + int initCallback(int videoFormat, int width, int height, int redrawRate, void* context, int drFlags) noexcept { + return FFMpegDecoder::instance().Init(videoFormat, width, height, redrawRate, context, drFlags); } - FFMpegDecoder* getDecoderInstance() { - return instance; - } - - int initCallback(int videoFormat, int width, int height, int redrawRate, void* context, int drFlags) { - return instance->Init(videoFormat, width, height, redrawRate, context, drFlags); - } - void startCallback() { - instance->Start(); - } - void stopCallback() { - instance->Stop(); - } - void cleanupCallback() { - instance->Cleanup(); - delete instance; - instance = NULL; + void cleanupCallback()noexcept { + FFMpegDecoder::instance().Cleanup(); } - FFMpegDecoder* FFMpegDecoder::getInstance() { - return instance; + int submitDecodeUnit(PDECODE_UNIT decodeUnit) noexcept { + return FFMpegDecoder::instance().SubmitDecodeUnit(decodeUnit); } DECODER_RENDERER_CALLBACKS FFMpegDecoder::getDecoder() { - instance = FFMpegDecoder::getInstance(); DECODER_RENDERER_CALLBACKS decoder_callbacks_sdl; LiInitializeVideoCallbacks(&decoder_callbacks_sdl); decoder_callbacks_sdl.setup = initCallback; - decoder_callbacks_sdl.start = startCallback; - decoder_callbacks_sdl.stop = stopCallback; decoder_callbacks_sdl.cleanup = cleanupCallback; - decoder_callbacks_sdl.submitDecodeUnit = NULL; - decoder_callbacks_sdl.capabilities = CAPABILITY_PULL_RENDERER | CAPABILITY_INTRA_REFRESH; + decoder_callbacks_sdl.submitDecodeUnit = submitDecodeUnit; + decoder_callbacks_sdl.capabilities = CAPABILITY_DIRECT_SUBMIT | CAPABILITY_INTRA_REFRESH; + //decoder_callbacks_sdl.capabilities = CAPABILITY_DIRECT_SUBMIT | CAPABILITY_REFERENCE_FRAME_INVALIDATION_HEVC; return decoder_callbacks_sdl; } } +void FFMpegDecoder::WaitForFrame() { + if (m_Pacer) { + m_Pacer->waitForFrame(); + } +} + +bool FFMpegDecoder::RenderFrameOnMainThread(std::shared_ptr& sceneRenderer) { + if (m_Pacer) { + return m_Pacer->renderOnMainThread(sceneRenderer); + } + return false; +} diff --git a/Streaming/FFmpegDecoder.h b/Streaming/FFmpegDecoder.h index dc6c6356..5e829441 100644 --- a/Streaming/FFmpegDecoder.h +++ b/Streaming/FFmpegDecoder.h @@ -3,6 +3,8 @@ #include #include #include "../Common/StepTimer.h" +#include "Pacer.h" +#include "VideoRenderer.h" extern "C" { #include @@ -18,39 +20,37 @@ namespace moonlight_xbox_dx class FFMpegDecoder { public: - FFMpegDecoder(std::shared_ptr r) { - resources = r; - }; + // Singleton accessor + static FFMpegDecoder &instance(); + + void CompleteInitialization(const std::shared_ptr& res, STREAM_CONFIGURATION *config); int Init(int videoFormat, int width, int height, int redrawRate, void* context, int drFlags); - void Start(); - void Stop(); void Cleanup(); - bool SubmitDU(); - AVFrame* GetFrame(); + int SubmitDecodeUnit(PDECODE_UNIT decodeUnit); int ModifyFrameDropTarget(bool increase); + void WaitForFrame(); + bool RenderFrameOnMainThread(std::shared_ptr& sceneRenderer); static FFMpegDecoder* getInstance(); static DECODER_RENDERER_CALLBACKS getDecoder(); - static FFMpegDecoder* createDecoderInstance(std::shared_ptr resources); std::recursive_mutex mutex; - bool shouldUnlock = false; bool hackWait = false; - int videoFormat,width,height; + int videoFormat, width, height; private: - int Decode(unsigned char* indata, int inlen); - AVPacket pkt; + FFMpegDecoder(); + FFMpegDecoder(const FFMpegDecoder &) = delete; + FFMpegDecoder &operator=(const FFMpegDecoder &) = delete; + const AVCodec* decoder; AVCodecContext* decoder_ctx; AVHWDeviceContext* device_ctx; AVD3D11VADeviceContext* d3d11va_device_ctx; unsigned char* ffmpeg_buffer; - int dec_frames_cnt; - AVFrame** dec_frames; - AVFrame** ready_frames; - int next_frame, current_frame; + int ffmpeg_buffer_size; std::shared_ptr resources; int64_t decodeStartTime; std::atomic_int frameDropTarget; - uint32_t m_LastFrameNumber; + int m_LastFrameNumber; + std::unique_ptr m_Pacer; }; } diff --git a/Streaming/Pacer.cpp b/Streaming/Pacer.cpp new file mode 100644 index 00000000..e792152d --- /dev/null +++ b/Streaming/Pacer.cpp @@ -0,0 +1,174 @@ +#include "pch.h" +#include "Pacer.h" +#include "../Plot/ImGuiPlots.h" +#include "Utils.hpp" + +// Limit the number of queued frames to prevent excessive memory consumption +// if the V-Sync source or renderer is blocked for a while. It's important +// that the sum of all queued frames between both pacing and rendering queues +// must not exceed the number buffer pool size to avoid running the decoder +// out of available decoding surfaces. +#define MAX_QUEUED_FRAMES 4 + +Pacer::Pacer(const std::shared_ptr& res) : + m_DeviceResources(res), + m_Stopping(false), + m_MaxVideoFps(0), + m_DisplayFps(0.0f) +{ + +} + +Pacer::~Pacer() +{ + m_Stopping = true; + m_RenderQueueNotEmpty.notify_all(); + + // Delete any remaining unconsumed frames + while (!m_RenderQueue.empty()) { + AVFrame* frame = std::move(m_RenderQueue.front()); + m_RenderQueue.pop_front(); + av_frame_free(&frame); + } +} + +void Pacer::waitForFrame() +{ + std::unique_lock lock(m_FrameQueueLock); + m_RenderQueueNotEmpty.wait(lock, [this]{ + return m_Stopping || !m_RenderQueue.empty(); + }); +} + +bool Pacer::renderOnMainThread(std::shared_ptr& sceneRenderer) +{ + AVFrame* frame = nullptr; + + { + std::scoped_lock lock(m_FrameQueueLock); + if (!m_RenderQueue.empty()) { + frame = std::move(m_RenderQueue.front()); + m_RenderQueue.pop_front(); + + FQLog("[<- pts %llu] dequeue frame, queue size %d\n", + frame->pts, m_RenderQueue.size()); + } + } + + if (!frame) { + return false; // no frame, don't Present() + } + + renderFrame(sceneRenderer, frame); + return true; // ok to Present() +} + +void Pacer::initialize(int maxVideoFps, double refreshRate) +{ + m_MaxVideoFps = maxVideoFps; + m_DisplayFps = refreshRate; + + Utils::Logf("Frame pacing: target %.2f Hz with %d FPS stream", + m_DisplayFps, m_MaxVideoFps); +} + +void Pacer::renderFrame(std::shared_ptr& sceneRenderer, AVFrame* frame) +{ + // Count time spent in Pacer's queues + LARGE_INTEGER beforeRender, afterRender; + QueryPerformanceCounter(&beforeRender); + + // Render it + sceneRenderer->Render(frame); + QueryPerformanceCounter(&afterRender); + + m_DeviceResources->GetStats()->SubmitPacerTime( + beforeRender.QuadPart - frame->pkt_dts, // pkt_dts is afterDecode.QuadPart + afterRender.QuadPart - beforeRender.QuadPart); + + av_frame_free(&frame); + + // Drop frames if we have too many queued up for a while + m_FrameQueueLock.lock(); + + int frameDropTarget = 0; + for (int queueHistoryEntry : m_RenderQueueHistory) { + if (queueHistoryEntry == 0) { + // Be lenient as long as the queue length + // resolves before the end of frame history + frameDropTarget = 2; + break; + } + } + + // Keep a rolling 500 ms window of render queue history + if (m_RenderQueueHistory.size() == m_MaxVideoFps / 2) { + m_RenderQueueHistory.pop_front(); + } + + m_RenderQueueHistory.push_back((int)m_RenderQueue.size()); + + // Catch up if we're several frames ahead + int dropCount = 0; + while (m_RenderQueue.size() > frameDropTarget) { + ++dropCount; + AVFrame* frame = std::move(m_RenderQueue.front()); + m_RenderQueue.pop_front(); + + Utils::Logf("pacer queue %d > frame drop target %d, dropping oldest frame (pts %lld)\n", + m_RenderQueue.size() + 1, frameDropTarget, frame->pts); + m_DeviceResources->GetStats()->SubmitDroppedFrame(1); + + // Drop the lock while we call av_frame_free() + m_FrameQueueLock.unlock(); + av_frame_free(&frame); + m_FrameQueueLock.lock(); + } + + m_FrameQueueLock.unlock(); + + // Don't graph 0 here, because it's already being done in dropFrameForEnqueue + if (dropCount > 0) { + ImGuiPlots::instance().observeFloat(PLOT_DROPPED_PACER, (float)dropCount); + } +} + +void Pacer::dropFrameForEnqueue(std::deque& queue) +{ + assert(queue.size() <= MAX_QUEUED_FRAMES); + + int dropCount = 0; + if (queue.size() == MAX_QUEUED_FRAMES) { + AVFrame* frame = std::move(queue.front()); + queue.pop_front(); + + Utils::Logf("pacer queue full (%d), dropping oldest frame (pts %lld)\n", + queue.size() + 1, frame->pts); + m_DeviceResources->GetStats()->SubmitDroppedFrame(1); + + av_frame_free(&frame); + dropCount = 1; + } + + ImGuiPlots::instance().observeFloat(PLOT_DROPPED_PACER, (float)dropCount); +} + +void Pacer::submitFrame(AVFrame* frame) +{ + // Make sure initialize() has been called + assert(m_MaxVideoFps != 0); + + // Queue the frame and possibly wake up the render thread + { + std::scoped_lock lock(m_FrameQueueLock); + dropFrameForEnqueue(m_RenderQueue); + m_RenderQueue.push_back(frame); + + size_t queueSize = m_RenderQueue.size(); + ImGuiPlots::instance().observeFloat(PLOT_QUEUED_FRAMES, (float)queueSize); + FQLog("[-> pts %lld] enqueue frame, render queue size %d\n", frame->pts, queueSize); + } + + // notify render loop of new frame + m_RenderQueueNotEmpty.notify_one(); +} diff --git a/Streaming/Pacer.h b/Streaming/Pacer.h new file mode 100644 index 00000000..d192b491 --- /dev/null +++ b/Streaming/Pacer.h @@ -0,0 +1,42 @@ +#pragma once + +#include "VideoRenderer.h" +#include +#include + +extern "C" +{ +#include +} + +using namespace moonlight_xbox_dx; + +class Pacer +{ +public: + Pacer(const std::shared_ptr& res); + + ~Pacer(); + + void submitFrame(AVFrame* frame); + + void initialize(int maxVideoFps, double refreshRate); + + void waitForFrame(); + + bool renderOnMainThread(std::shared_ptr& sceneRenderer); + +private: + void renderFrame(std::shared_ptr& sceneRenderer, AVFrame* frame); + + void dropFrameForEnqueue(std::deque& queue); + + std::shared_ptr m_DeviceResources; + std::deque m_RenderQueue; + std::deque m_RenderQueueHistory; + std::condition_variable m_RenderQueueNotEmpty; + std::mutex m_FrameQueueLock; + std::atomic m_Stopping{false}; + int m_MaxVideoFps; + double m_DisplayFps; +}; diff --git a/Streaming/StatsRenderer.cpp b/Streaming/StatsRenderer.cpp index a1886102..2e1c5977 100644 --- a/Streaming/StatsRenderer.cpp +++ b/Streaming/StatsRenderer.cpp @@ -73,14 +73,15 @@ void StatsRenderer::RenderGraphs() QueryPerformanceCounter(&t0); // we malloc a buffer for each stat only once and reuse it each frame - assert(PlotCount == 6); - static float *buffers[6] = { + assert(PlotCount == 7); + static float *buffers[7] = { (float *)malloc(sizeof(float) * 512), (float *)malloc(sizeof(float) * 512), (float *)malloc(sizeof(float) * 512), (float *)malloc(sizeof(float) * 512), (float *)malloc(sizeof(float) * 512), - (float *)malloc(sizeof(float) * 512)}; + (float *)malloc(sizeof(float) * 512), + (float *)malloc(sizeof(float) * 512)}; ImGuiIO &io = ImGui::GetIO(); @@ -96,7 +97,7 @@ void StatsRenderer::RenderGraphs() float itemSpacingX = ImGui::GetStyle().ItemSpacing.x; float itemSpacingY = ImGui::GetStyle().ItemSpacing.y; float row1Width = (3 * graphW) + (2 * itemSpacingX); - float totalHeight = (2 * graphH) + (2 * itemSpacingY) + 50; + float totalHeight = (3 * graphH) + (2 * itemSpacingY) + 50; // Anchor window to top-right ImVec2 windowPos(m_displayWidth - 10.0f, 10.0f); // 10px margin @@ -172,6 +173,10 @@ void StatsRenderer::RenderGraphs() draw_plot(row2[c], graphW, graphH); } + // 3rd row for quickly graphing something if needed + // ImGui::Dummy(ImVec2(1.0f, itemSpacingY)); + // draw_plot(PLOT_ETC, graphW, graphH); + ImGui::End(); LARGE_INTEGER t1; diff --git a/Streaming/VideoRenderer.cpp b/Streaming/VideoRenderer.cpp index 91e14de2..e18ff6ce 100644 --- a/Streaming/VideoRenderer.cpp +++ b/Streaming/VideoRenderer.cpp @@ -101,22 +101,19 @@ void VideoRenderer::Update(DX::StepTimer const& timer) bool renderedOneFrame = false; // Renders one frame using the vertex and pixel shaders. -bool VideoRenderer::Render() +bool VideoRenderer::Render(AVFrame* frame) { // Loading is asynchronous. Only draw geometry after it's loaded. if (!m_loadingComplete) { return true; } - //Create a rendering texture - FFMpegDecoder::getInstance()->shouldUnlock = false; - if (!FFMpegDecoder::getInstance()->SubmitDU()) { - return false; - } - AVFrame* frame = FFMpegDecoder::getInstance()->GetFrame(); - if (frame == nullptr) { - return false; - } + + FQLog("[%lld ms] rendering frame\n", frame->pts); + + auto d3dcontext = m_deviceResources->GetD3DDeviceContext(); + auto d3ddevice = m_deviceResources->GetD3DDevice(); + Microsoft::WRL::ComPtr renderTexture; D3D11_BOX box; box.left = 0; @@ -125,43 +122,33 @@ bool VideoRenderer::Render() box.bottom = renderTextureDesc.Height; box.front = 0; box.back = 1; - if (frame->format != AV_PIX_FMT_D3D11) { - char msg[2048]; - sprintf_s(msg, "Pixel format mismatch - got %d instead of D3D11!\n", frame->format); - Utils::Log(msg); - DX::ThrowIfFailed(m_deviceResources->GetD3DDevice()->CreateTexture2D(&renderTextureDesc, NULL, renderTexture.GetAddressOf()), "Render Texture Creation"); - } - else { - FFMpegDecoder::getInstance()->mutex.lock(); - FFMpegDecoder::getInstance()->shouldUnlock = true; - ID3D11Texture2D* ffmpegTexture = (ID3D11Texture2D*)(frame->data[0]); - D3D11_TEXTURE2D_DESC ffmpegDesc; - ffmpegTexture->GetDesc(&ffmpegDesc); - int index = (int)(frame->data[1]); - box.right = std::min(renderTextureDesc.Width, ffmpegDesc.Width); - box.bottom = std::min(renderTextureDesc.Height, ffmpegDesc.Height); - renderTextureDesc.Format = ffmpegDesc.Format; - DX::ThrowIfFailed(m_deviceResources->GetD3DDevice()->CreateTexture2D(&renderTextureDesc, NULL, renderTexture.GetAddressOf()), "Render Texture Creation"); - m_deviceResources->GetD3DDeviceContext()->CopySubresourceRegion(renderTexture.Get(), 0, 0, 0, 0, ffmpegTexture, index, &box); - } - auto context = m_deviceResources->GetD3DDeviceContext(); + ID3D11Texture2D* ffmpegTexture = (ID3D11Texture2D*)(frame->data[0]); + D3D11_TEXTURE2D_DESC ffmpegDesc; + ffmpegTexture->GetDesc(&ffmpegDesc); + int index = (int)(frame->data[1]); + box.right = std::min(renderTextureDesc.Width, ffmpegDesc.Width); + box.bottom = std::min(renderTextureDesc.Height, ffmpegDesc.Height); + renderTextureDesc.Format = ffmpegDesc.Format; + DX::ThrowIfFailed(d3ddevice->CreateTexture2D(&renderTextureDesc, NULL, renderTexture.GetAddressOf()), "Render Texture Creation"); + d3dcontext->CopySubresourceRegion(renderTexture.Get(), 0, 0, 0, 0, ffmpegTexture, index, &box); + UINT stride = sizeof(VERTEX); UINT offset = 0; - context->IASetIndexBuffer(m_indexBuffer.Get(), + d3dcontext->IASetIndexBuffer(m_indexBuffer.Get(), DXGI_FORMAT_R32_UINT, 0); - context->IASetVertexBuffers( + d3dcontext->IASetVertexBuffers( 0, 1, m_vertexBuffer.GetAddressOf(), &stride, &offset ); - context->IASetPrimitiveTopology(D3D11_PRIMITIVE_TOPOLOGY_TRIANGLELIST); - context->IASetInputLayout(m_inputLayout.Get()); + d3dcontext->IASetPrimitiveTopology(D3D11_PRIMITIVE_TOPOLOGY_TRIANGLELIST); + d3dcontext->IASetInputLayout(m_inputLayout.Get()); // Attach our vertex shader. - context->VSSetShader(m_vertexShader.Get(), nullptr, 0); + d3dcontext->VSSetShader(m_vertexShader.Get(), nullptr, 0); Microsoft::WRL::ComPtr m_luminance_shader_resource_view; Microsoft::WRL::ComPtr m_chrominance_shader_resource_view; @@ -170,7 +157,7 @@ bool VideoRenderer::Render() D3D11_SRV_DIMENSION_TEXTURE2D, (renderTextureDesc.Format == DXGI_FORMAT_P010) ? DXGI_FORMAT_R16_UNORM : DXGI_FORMAT_R8_UNORM ); - DX::ThrowIfFailed(m_deviceResources->GetD3DDevice()->CreateShaderResourceView( + DX::ThrowIfFailed(d3ddevice->CreateShaderResourceView( renderTexture.Get(), &luminance_desc, m_luminance_shader_resource_view.GetAddressOf()), "Luminance SRV Creation" ); @@ -179,15 +166,15 @@ bool VideoRenderer::Render() D3D11_SRV_DIMENSION_TEXTURE2D, (renderTextureDesc.Format == DXGI_FORMAT_P010) ? DXGI_FORMAT_R16G16_UNORM : DXGI_FORMAT_R8G8_UNORM ); - DX::ThrowIfFailed(m_deviceResources->GetD3DDevice()->CreateShaderResourceView( + DX::ThrowIfFailed(d3ddevice->CreateShaderResourceView( renderTexture.Get(), &chrominance_desc, m_chrominance_shader_resource_view.GetAddressOf()), "Chrominance SRV Creation" ); - m_deviceResources->GetD3DDeviceContext()->PSSetShaderResources(0, 1, m_luminance_shader_resource_view.GetAddressOf()); - m_deviceResources->GetD3DDeviceContext()->PSSetShaderResources(1, 1, m_chrominance_shader_resource_view.GetAddressOf()); + d3dcontext->PSSetShaderResources(0, 1, m_luminance_shader_resource_view.GetAddressOf()); + d3dcontext->PSSetShaderResources(1, 1, m_chrominance_shader_resource_view.GetAddressOf()); this->bindColorConversion(frame); - m_deviceResources->GetD3DDeviceContext()->DrawIndexed(6, 0, 0); + d3dcontext->DrawIndexed(6, 0, 0); if (frame->color_trc != m_LastColorTrc) { DXGI_COLOR_SPACE_TYPE colorspace = {}; @@ -566,14 +553,7 @@ void VideoRenderer::bindColorConversion(AVFrame* frame) void VideoRenderer::SetHDR(bool enabled) { - HRESULT hr; - - // Do not try to render any frames while we're changing display modes - auto ffmpeg = FFMpegDecoder::getInstance(); - if (ffmpeg != nullptr) { - ffmpeg->mutex.lock(); - ffmpeg->shouldUnlock = false; - } + FFMpegDecoder::instance().mutex.lock(); if (enabled) { SS_HDR_METADATA sunshineHdrMetadata; @@ -591,12 +571,10 @@ void VideoRenderer::SetHDR(bool enabled) client->SetDisplayHDR(false, SS_HDR_METADATA{}); } - if (ffmpeg != nullptr) { - ffmpeg->shouldUnlock = true; - ffmpeg->mutex.unlock(); - } + FFMpegDecoder::instance().mutex.unlock(); } void VideoRenderer::Stop() { // nothing to do } + diff --git a/Streaming/VideoRenderer.h b/Streaming/VideoRenderer.h index 7f8acce4..e6c9b1a7 100644 --- a/Streaming/VideoRenderer.h +++ b/Streaming/VideoRenderer.h @@ -5,6 +5,10 @@ #include "State\MoonlightClient.h" #include "State\StreamConfiguration.h" +extern "C" { +#include +} + namespace moonlight_xbox_dx { @@ -26,7 +30,7 @@ namespace moonlight_xbox_dx void Update(DX::StepTimer const& timer); void scaleSourceToDestinationSurface(IRECT* src, IRECT* dst); void screenSpaceToNormalizedDeviceCoords(IRECT* src, FRECT* dst, int viewportWidth, int viewportHeight); - bool Render(); + bool Render(AVFrame* frame); void bindColorConversion(AVFrame* frame); void SetHDR(bool enabled); void Stop(); diff --git a/Streaming/moonlight_xbox_dxMain.cpp b/Streaming/moonlight_xbox_dxMain.cpp index 822ea66f..9c7f7465 100644 --- a/Streaming/moonlight_xbox_dxMain.cpp +++ b/Streaming/moonlight_xbox_dxMain.cpp @@ -4,6 +4,7 @@ #include "../Plot/ImGuiPlots.h" #include "Utils.hpp" #include +#include using namespace Windows::Gaming::Input; @@ -40,7 +41,7 @@ moonlight_xbox_dxMain::moonlight_xbox_dxMain(const std::shared_ptrRegisterDeviceNotify(this); - m_sceneRenderer = std::make_unique(m_deviceResources, moonlightClient, configuration); + m_sceneRenderer = std::make_shared(m_deviceResources, moonlightClient, configuration); m_LogRenderer = std::make_unique(m_deviceResources); @@ -111,13 +112,15 @@ void moonlight_xbox_dxMain::StartRenderLoop() while (action->Status == AsyncStatus::Started) { critical_section::scoped_lock lock(m_criticalSection); - LARGE_INTEGER beforeRender, beforePresent; + LARGE_INTEGER beforeWait, afterWait, beforePresent; + FFMpegDecoder::instance().WaitForFrame(); Update(); - QueryPerformanceCounter(&beforeRender); + + // Make sure our use of D3D doesn't overlap with ffmpeg's hardware decoding + FFMpegDecoder::instance().mutex.lock(); if (Render()) { QueryPerformanceCounter(&beforePresent); - m_stats->SubmitRenderTime(beforePresent.QuadPart - beforeRender.QuadPart); static uint64_t lastPresentTime = 0; if (lastPresentTime > 0) { @@ -128,6 +131,7 @@ void moonlight_xbox_dxMain::StartRenderLoop() m_deviceResources->Present(); } + FFMpegDecoder::instance().mutex.unlock(); } }); m_renderLoopWorker = ThreadPool::RunAsync(workItemHandler, WorkItemPriority::High, WorkItemOptions::TimeSliced); @@ -375,7 +379,7 @@ bool moonlight_xbox_dxMain::Render() } // Render the scene objects. - bool shouldPresent = m_sceneRenderer->Render(); + bool shouldPresent = FFMpegDecoder::instance().RenderFrameOnMainThread(m_sceneRenderer); m_LogRenderer->Render(); m_statsTextRenderer->Render(showImGui); diff --git a/Streaming/moonlight_xbox_dxMain.h b/Streaming/moonlight_xbox_dxMain.h index eed1e92d..cfb60a26 100644 --- a/Streaming/moonlight_xbox_dxMain.h +++ b/Streaming/moonlight_xbox_dxMain.h @@ -45,7 +45,7 @@ namespace moonlight_xbox_dx // Cached pointer to device resources. std::shared_ptr m_deviceResources; - std::unique_ptr m_sceneRenderer; + std::shared_ptr m_sceneRenderer; std::unique_ptr m_LogRenderer; std::unique_ptr m_statsTextRenderer; diff --git a/moonlight-xbox-dx.vcxproj b/moonlight-xbox-dx.vcxproj index 15545848..d0a7a906 100644 --- a/moonlight-xbox-dx.vcxproj +++ b/moonlight-xbox-dx.vcxproj @@ -344,6 +344,8 @@ + + @@ -477,6 +479,7 @@ + diff --git a/moonlight-xbox-dx.vcxproj.filters b/moonlight-xbox-dx.vcxproj.filters index 7a85f08e..7cd00399 100644 --- a/moonlight-xbox-dx.vcxproj.filters +++ b/moonlight-xbox-dx.vcxproj.filters @@ -380,6 +380,9 @@ Source Files + + Source Files + @@ -463,6 +466,12 @@ Header Files + + Header Files + + + Header Files + @@ -2402,6 +2411,339 @@ DLLs + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + diff --git a/pch.h b/pch.h index ad68b8e9..1c9d7934 100644 --- a/pch.h +++ b/pch.h @@ -85,3 +85,19 @@ static double QpcToMs(uint64_t qpc) { Utils::Logf(fmt, ##__VA_ARGS__); \ }); \ } while (0) + +// Frame queue debugging +#if !defined(NDEBUG) +//#define FRAME_QUEUE_VERBOSE +#endif + +#ifdef FRAME_QUEUE_VERBOSE + #define FQLog(fmt, ...) \ + moonlight_xbox_dx::Utils::Logf("[%d] " fmt, ::GetCurrentThreadId(), ##__VA_ARGS__) +#else + #if defined(_MSC_VER) + #define FQLog(...) __noop + #else + #define FQLog(fmt, ...) do {} while(0) + #endif +#endif From 58c8739e6c82a74434d18f896251f9fa5f7e8a78 Mon Sep 17 00:00:00 2001 From: Andy Grundman Date: Sun, 14 Sep 2025 10:59:18 -0400 Subject: [PATCH 03/22] Slow down the ProcessInput loop to only poll about every 1ms --- Streaming/moonlight_xbox_dxMain.cpp | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/Streaming/moonlight_xbox_dxMain.cpp b/Streaming/moonlight_xbox_dxMain.cpp index 9c7f7465..f8729dcb 100644 --- a/Streaming/moonlight_xbox_dxMain.cpp +++ b/Streaming/moonlight_xbox_dxMain.cpp @@ -140,10 +140,27 @@ void moonlight_xbox_dxMain::StartRenderLoop() } auto inputItemHandler = ref new WorkItemHandler([this](IAsyncAction^ action) { + // Target period = 1000hz + using clock = std::chrono::high_resolution_clock; + const auto period = std::chrono::microseconds(1000); + auto next = clock::now() + period; + // Calculate the updated frame and render once per vertical blanking interval. while (action->Status == AsyncStatus::Started) { ProcessInput(); + + auto now = clock::now(); + next += period; + if (now > next) { + // If we missed the deadline, catch up + auto late = now - next; + auto missed = (late + period - std::chrono::microseconds(1)) / period; + next += missed * period; + } + else { + std::this_thread::sleep_until(next); + } } }); From fd1c2291c18b139ca1725d1bb980ba18a9828074 Mon Sep 17 00:00:00 2001 From: Andy Grundman Date: Sun, 14 Sep 2025 15:23:20 -0400 Subject: [PATCH 04/22] Trying ProcessInput at 2ms, removed usleep --- Streaming/moonlight_xbox_dxMain.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Streaming/moonlight_xbox_dxMain.cpp b/Streaming/moonlight_xbox_dxMain.cpp index f8729dcb..6233294d 100644 --- a/Streaming/moonlight_xbox_dxMain.cpp +++ b/Streaming/moonlight_xbox_dxMain.cpp @@ -142,7 +142,7 @@ void moonlight_xbox_dxMain::StartRenderLoop() { // Target period = 1000hz using clock = std::chrono::high_resolution_clock; - const auto period = std::chrono::microseconds(1000); + const auto period = std::chrono::microseconds(2000); auto next = clock::now() + period; // Calculate the updated frame and render once per vertical blanking interval. @@ -370,7 +370,6 @@ void moonlight_xbox_dxMain::ProcessInput() moonlightClient->SendGamepadReading(i, reading); } previousReading[i] = reading; - usleep(2000); } } From 6e6c2bea512e378425d09b99dc59ccd44ad89081 Mon Sep 17 00:00:00 2001 From: Andy Grundman Date: Mon, 15 Sep 2025 03:36:11 -0400 Subject: [PATCH 05/22] The single lock around most of the render loop hurts decoder performance. Try to make the locks a bit more granular. --- Common/DeviceResources.cpp | 5 ++ Streaming/FFmpegDecoder.cpp | 78 +++++++++++++++++------------ Streaming/FFmpegDecoder.h | 20 +++++++- Streaming/moonlight_xbox_dxMain.cpp | 18 ++++++- pch.h | 2 +- 5 files changed, 87 insertions(+), 36 deletions(-) diff --git a/Common/DeviceResources.cpp b/Common/DeviceResources.cpp index 4d8e5a97..62799ca0 100644 --- a/Common/DeviceResources.cpp +++ b/Common/DeviceResources.cpp @@ -652,6 +652,9 @@ void DX::DeviceResources::Present() { HRESULT hr = E_FAIL; DXGI_PRESENT_PARAMETERS parameters = { 0 }; + + LOCK_D3D("Present"); + if (m_enableVsync) { // This will still use vsync due to the lack of DXGI_PRESENT_ALLOW_TEARING hr = m_swapChain->Present1(0, 0, ¶meters); @@ -673,6 +676,8 @@ void DX::DeviceResources::Present() // Discard the contents of the depth stencil. m_d3dContext->DiscardView1(m_d3dDepthStencilView.Get(), nullptr, 0); + UNLOCK_D3D(); + // If the device was removed either by a disconnection or a driver upgrade, we // must recreate all device resources. if (hr == DXGI_ERROR_DEVICE_REMOVED || hr == DXGI_ERROR_DEVICE_RESET) diff --git a/Streaming/FFmpegDecoder.cpp b/Streaming/FFmpegDecoder.cpp index 4266c0bf..976663a6 100644 --- a/Streaming/FFmpegDecoder.cpp +++ b/Streaming/FFmpegDecoder.cpp @@ -27,7 +27,7 @@ static bool ensure_buf_size(unsigned char **buf, int *buf_size, int required_siz if (*buf_size >= required_size) return true; - Utils::Logf("ensure_buf_size grew from %d -> %d\n", *buf_size, required_size); + FQLog("ensure_buf_size grew from %d -> %d\n", *buf_size, required_size); *buf_size = required_size; *buf = (unsigned char *)realloc(*buf, *buf_size); @@ -55,20 +55,17 @@ namespace moonlight_xbox_dx { ffmpeg_buffer(nullptr), ffmpeg_buffer_size(0), resources(nullptr), - decodeStartTime(0), frameDropTarget(2), m_LastFrameNumber(0), m_Pacer(nullptr) { } - void lock_context(void* dec) { - auto ff = (FFMpegDecoder*)dec; - ff->mutex.lock(); + void lock_context(void*) { + LOCK_D3D("ffmpeg"); } - void unlock_context(void* dec) { - auto ff = (FFMpegDecoder*)dec; - ff->mutex.unlock(); + void unlock_context(void*) { + UNLOCK_D3D(); } void ffmpeg_log_callback(void *ptr, int level, const char *fmt, va_list vl) { @@ -142,7 +139,7 @@ namespace moonlight_xbox_dx { d3d11va_device_ctx->device_context = this->resources->GetD3DDeviceContext(); d3d11va_device_ctx->lock = lock_context; d3d11va_device_ctx->unlock = unlock_context; - d3d11va_device_ctx->lock_ctx = this; + d3d11va_device_ctx->lock_ctx = nullptr; int err2; if ((err2 = av_hwdevice_ctx_init(hw_device_ctx)) < 0) { char msg[2048]; @@ -152,13 +149,15 @@ namespace moonlight_xbox_dx { } - decoder_ctx->hw_device_ctx = av_buffer_ref(hw_device_ctx); - decoder_ctx->pix_fmt = AV_PIX_FMT_D3D11; - decoder_ctx->sw_pix_fmt = (videoFormat & VIDEO_FORMAT_MASK_10BIT) ? AV_PIX_FMT_P010 : AV_PIX_FMT_NV12; - decoder_ctx->width = width; - decoder_ctx->height = height; + decoder_ctx->hw_device_ctx = av_buffer_ref(hw_device_ctx); + decoder_ctx->pix_fmt = AV_PIX_FMT_D3D11; + decoder_ctx->sw_pix_fmt = (videoFormat & VIDEO_FORMAT_MASK_10BIT) ? AV_PIX_FMT_P010 : AV_PIX_FMT_NV12; + decoder_ctx->pkt_timebase.num = 1; + decoder_ctx->pkt_timebase.den = 90000; + decoder_ctx->width = width; + decoder_ctx->height = height; - int err = avcodec_open2(decoder_ctx, decoder, NULL); + int err = avcodec_open2(decoder_ctx, decoder, NULL); if (err < 0) { char msg[2048]; sprintf(msg, "Failed to create FFMpeg Codec: %d\n", err); @@ -204,9 +203,12 @@ namespace moonlight_xbox_dx { // Called by the VideoDec thread int FFMpegDecoder::SubmitDecodeUnit(PDECODE_UNIT decodeUnit) { + LARGE_INTEGER decodeStart, decodeEnd; PLENTRY entry = decodeUnit->bufferList; int length = 0; - int64_t decodeStart = av_gettime_relative(); + QueryPerformanceCounter(&decodeStart); + + FQLog("*** SubmitDecodeUnit(frame=%d, length=%d)\n", decodeUnit->frameNumber, decodeUnit->fullLength); if (!ensure_buf_size(&ffmpeg_buffer, &ffmpeg_buffer_size, decodeUnit->fullLength + AV_INPUT_BUFFER_PADDING_SIZE)) { Utils::Logf("Couldn't realloc ffmpeg_buffer\n"); @@ -223,7 +225,7 @@ namespace moonlight_xbox_dx { // Detect breaks in the frame sequence indicating dropped packets uint32_t droppedFramesNetwork = 0; if (m_LastFrameNumber > 0 && decodeUnit->frameNumber > (m_LastFrameNumber + 1)) { - // Any frame number greater than m_LastFrameNumber + 1 represents a dropped frame + // Any frame number greater than m_LastFrameNumber + 1 represents a dropped fraFme droppedFramesNetwork = decodeUnit->frameNumber - (m_LastFrameNumber + 1); } m_LastFrameNumber = decodeUnit->frameNumber; @@ -235,6 +237,9 @@ namespace moonlight_xbox_dx { AVPacket *pkt = av_packet_alloc(); pkt->data = ffmpeg_buffer; pkt->size = length; + pkt->pts = (int64_t)decodeUnit->rtpTimestamp; + pkt->dts = pkt->pts; + int err = avcodec_send_packet(decoder_ctx, pkt); av_packet_unref(pkt); if (err < 0) { @@ -243,19 +248,17 @@ namespace moonlight_xbox_dx { Utils::Logf("avcodec_send_packet failed: %s\n", ffmpegError); return DR_NEED_IDR; } + FQLog("avcodec_send_packet size %d\n", length); - for (;;) { + while (err >= 0) { AVFrame* frame = av_frame_alloc(); err = avcodec_receive_frame(decoder_ctx, frame); - if (err == AVERROR(EAGAIN)) { - av_frame_free(&frame); - break; // decoder needs more input before producing another frame - } - if (err == AVERROR_EOF) { - av_frame_free(&frame); - break; - } - if (err < 0) { + if (err == AVERROR(EAGAIN) || err == AVERROR_EOF) { + FQLog("avcodec_receive_frame returned %d\n", err); + av_frame_free(&frame); + break; + } + else if (err < 0) { char ffmpegError[1024]; av_strerror(err, ffmpegError, sizeof(ffmpegError)); Utils::Logf("avcodec_receive_frame failed: %s\n", ffmpegError); @@ -264,18 +267,29 @@ namespace moonlight_xbox_dx { } // Capture a frame timestamp to measuring pacing delay - LARGE_INTEGER(afterDecode); - QueryPerformanceCounter(&afterDecode); - frame->pkt_dts = afterDecode.QuadPart; + QueryPerformanceCounter(&decodeEnd); + //frame->pkt_dts = decodeEnd.QuadPart; // Store the presentation time - frame->pts = decodeUnit->presentationTimeMs; + //frame->pts = decodeUnit->presentationTimeMs; // Queue the frame for rendering (or render now if pacer is disabled) m_Pacer->submitFrame(frame); + + double decodeTimeMs = QpcToMs(decodeEnd.QuadPart - decodeStart.QuadPart); + FQLog("avcodec_receive_frame got frame %3"PRId64" in %.3f ms (pts %llu)\n", decoder_ctx->frame_num, decodeTimeMs, frame->pts); + // if (decodeUnit->frameNumber > 200 && decodeTimeMs > 12.0f) { + // __debugbreak; + // assert(false); + // } + + // Even though we have a valid frame, the ffmpeg API needs us to loop and call avcodec_receive_frame() + // again where we expect to get AVERROR(EAGAIN) and break out. } - this->resources->GetStats()->SubmitDecodeMs(double(av_gettime_relative() - decodeStart) / 1000.0); + if (decodeEnd.QuadPart > decodeStart.QuadPart) { + this->resources->GetStats()->SubmitDecodeMs(QpcToMs(decodeEnd.QuadPart - decodeStart.QuadPart)); + } // Restore default log level after a successful decode if (av_log_get_level() > 0) { diff --git a/Streaming/FFmpegDecoder.h b/Streaming/FFmpegDecoder.h index 5e829441..7e2e3635 100644 --- a/Streaming/FFmpegDecoder.h +++ b/Streaming/FFmpegDecoder.h @@ -5,6 +5,7 @@ #include "../Common/StepTimer.h" #include "Pacer.h" #include "VideoRenderer.h" +#include "Utils.hpp" extern "C" { #include @@ -48,9 +49,26 @@ namespace moonlight_xbox_dx unsigned char* ffmpeg_buffer; int ffmpeg_buffer_size; std::shared_ptr resources; - int64_t decodeStartTime; std::atomic_int frameDropTarget; int m_LastFrameNumber; std::unique_ptr m_Pacer; }; } + +inline void LOCK_D3D(const char *msg) { +#if !defined(NDEBUG) + LARGE_INTEGER t0, t1; + QueryPerformanceCounter(&t0); +#endif + + FFMpegDecoder::instance().mutex.lock(); + +#if !defined(NDEBUG) + QueryPerformanceCounter(&t1); + Utils::Logf("%s acquired lock_context in %.3f ms\n", msg, QpcToMs(t1.QuadPart - t0.QuadPart)); +#endif +} + +inline void UNLOCK_D3D() { + FFMpegDecoder::instance().mutex.unlock(); +} diff --git a/Streaming/moonlight_xbox_dxMain.cpp b/Streaming/moonlight_xbox_dxMain.cpp index 6233294d..d748a8b7 100644 --- a/Streaming/moonlight_xbox_dxMain.cpp +++ b/Streaming/moonlight_xbox_dxMain.cpp @@ -118,7 +118,12 @@ void moonlight_xbox_dxMain::StartRenderLoop() Update(); // Make sure our use of D3D doesn't overlap with ffmpeg's hardware decoding - FFMpegDecoder::instance().mutex.lock(); + // LARGE_INTEGER t0, t1; + // FQLog("render loop before lock_context\n"); + // QueryPerformanceCounter(&t0); + // FFMpegDecoder::instance().mutex.lock(); + // QueryPerformanceCounter(&t1); + // FQLog("render loop acquired lock_context in %.3f ms\n", QpcToMs(t1.QuadPart - t0.QuadPart)); if (Render()) { QueryPerformanceCounter(&beforePresent); @@ -131,7 +136,8 @@ void moonlight_xbox_dxMain::StartRenderLoop() m_deviceResources->Present(); } - FFMpegDecoder::instance().mutex.unlock(); + //FFMpegDecoder::instance().mutex.unlock(); + //FQLog("render loop unlock context\n"); } }); m_renderLoopWorker = ThreadPool::RunAsync(workItemHandler, WorkItemPriority::High, WorkItemOptions::TimeSliced); @@ -384,25 +390,33 @@ bool moonlight_xbox_dxMain::Render() return false; } + LOCK_D3D("Clear"); Clear(); + UNLOCK_D3D(); // Start the Dear ImGui frame, limited to only running at 60fps bool showImGui = m_deviceResources->GetShowImGui(); if (showImGui) { + LOCK_D3D("ImGui 1"); ImGui_ImplDX11_NewFrame(); + UNLOCK_D3D(); ImGui_ImplUwp_NewFrame(m_deviceResources->GetPixelWidth(), m_deviceResources->GetPixelHeight()); ImGui::NewFrame(); } // Render the scene objects. + LOCK_D3D("Render"); bool shouldPresent = FFMpegDecoder::instance().RenderFrameOnMainThread(m_sceneRenderer); m_LogRenderer->Render(); m_statsTextRenderer->Render(showImGui); + UNLOCK_D3D(); if (showImGui) { RenderImGui(); ImGui::Render(); + LOCK_D3D("ImGui 2"); ImGui_ImplDX11_RenderDrawData(ImGui::GetDrawData()); + UNLOCK_D3D(); } return shouldPresent; diff --git a/pch.h b/pch.h index 1c9d7934..9d62c590 100644 --- a/pch.h +++ b/pch.h @@ -88,7 +88,7 @@ static double QpcToMs(uint64_t qpc) { // Frame queue debugging #if !defined(NDEBUG) -//#define FRAME_QUEUE_VERBOSE +#define FRAME_QUEUE_VERBOSE #endif #ifdef FRAME_QUEUE_VERBOSE From d4822e7ad136b8ca14b9c26af5fbc042e7505402 Mon Sep 17 00:00:00 2001 From: Andy Grundman Date: Mon, 15 Sep 2025 08:29:47 -0400 Subject: [PATCH 06/22] It seems like we can remove all D3D locking by enabling SetMultithreadProtected mode. Also removed the force tearing setting (it's the same as disabling vsync), and changed the modify frame queue to tweak the Pacer target. --- .clang-format | 2 +- Common/DeviceResources.cpp | 48 ++-------------- Common/DeviceResources.h | 3 - Pages/AppPage.xaml.cpp | 1 - Pages/HostSettingsPage.xaml | 37 ++++-------- Pages/StreamPage.xaml.cpp | 5 +- State/ApplicationState.cpp | 2 - State/MoonlightHost.h | 12 ---- State/StreamConfiguration.h | 1 - Streaming/FFmpegDecoder.cpp | 68 ++++++++++++---------- Streaming/FFmpegDecoder.h | 87 +++++++++++++++-------------- Streaming/Pacer.cpp | 47 +++++++++++----- Streaming/Pacer.h | 3 + Streaming/VideoRenderer.cpp | 4 +- Streaming/moonlight_xbox_dxMain.cpp | 16 +++--- pch.h | 1 + 16 files changed, 149 insertions(+), 188 deletions(-) diff --git a/.clang-format b/.clang-format index d9a5afce..4782c9af 100644 --- a/.clang-format +++ b/.clang-format @@ -2,7 +2,7 @@ BasedOnStyle: LLVM UseTab: ForIndentation IndentWidth: 4 TabWidth: 4 -BreakBeforeBraces: Mozilla +BreakBeforeBraces: Attach ColumnLimit: 0 AllowShortIfStatementsOnASingleLine: true AllowShortLoopsOnASingleLine: true diff --git a/Common/DeviceResources.cpp b/Common/DeviceResources.cpp index 62799ca0..428adf66 100644 --- a/Common/DeviceResources.cpp +++ b/Common/DeviceResources.cpp @@ -59,13 +59,11 @@ namespace ScreenRotation DX::DeviceResources::DeviceResources() : m_backBufferFormat(DXGI_FORMAT_R10G10B10A2_UNORM), // 10-bit for HDR m_screenViewport(), - m_d3dFeatureLevel(D3D_FEATURE_LEVEL_10_0), + m_d3dFeatureLevel(D3D_FEATURE_LEVEL_11_1), m_d3dRenderTargetSize(), m_dxgiFactoryFlags(0), m_enableVsync(true), - m_frameLatencyWaitableObject(), m_swapchainVsync(true), - m_forceTearing(false), m_outputSize(), m_logicalSize(), m_nativeOrientation(DisplayOrientations::None), @@ -250,11 +248,9 @@ void DX::DeviceResources::CreateWindowSizeDependentResources() lround(m_d3dRenderTargetSize.Width), lround(m_d3dRenderTargetSize.Height), m_backBufferFormat, - 0 + m_enableVsync ? 0 : DXGI_SWAP_CHAIN_FLAG_ALLOW_TEARING ); - Utils::Logf("ResizeBuffers() ALLOW_TEARING=%d\n", (m_enableVsync && !m_forceTearing) ? 0 : 1); - if (hr == DXGI_ERROR_DEVICE_REMOVED || hr == DXGI_ERROR_DEVICE_RESET) { // If the device was removed for any reason, a new device and swap chain will need to be created. @@ -284,7 +280,7 @@ void DX::DeviceResources::CreateWindowSizeDependentResources() //Check moonlight-stream/moonlight-qt/app/streaming/video/ffmpeg-renderers/d3d11va.cpp for rationale swapChainDesc.BufferCount = 5; swapChainDesc.SwapEffect = DXGI_SWAP_EFFECT_FLIP_DISCARD; - swapChainDesc.Flags = 0; + swapChainDesc.Flags = m_enableVsync ? 0 : DXGI_SWAP_CHAIN_FLAG_ALLOW_TEARING; swapChainDesc.Scaling = DXGI_SCALING_STRETCH; swapChainDesc.AlphaMode = DXGI_ALPHA_MODE_IGNORE; @@ -315,30 +311,12 @@ void DX::DeviceResources::CreateWindowSizeDependentResources() ) ); - Utils::Logf("CreateSwapChainForComposition() ALLOW_TEARING=%d\n", (m_enableVsync && !m_forceTearing) ? 0 : 1); - - // XXX This seems like the better type of swap chain with fullscreen support - // TODO: review https://github.com/ahmed605/imgui-uwp/blob/master/examples/example_uwp_directx11/main.cpp - // DX::ThrowIfFailed( - // dxgiFactory->CreateSwapChainForCoreWindow( - // m_d3dDevice.Get(), - // // reinterpret_cast<::IUnknown*>(winrt::get_abi(CoreWindow::GetForCurrentThread())), // compiles and crashes - // // reinterpret_cast(CoreWindow::GetForCurrentThread()), // this also compiles and crashes - // winrt::get_unknown(CoreWindow::GetForCurrentThread()), - // &swapChainDesc, - // nullptr, - // &swapChain - // ) - // ); + Utils::Logf("Vsync: %s\n", m_enableVsync ? "enabled" : "disabled"); DX::ThrowIfFailed( swapChain.As(&m_swapChain) ); - // This controls how GetFrameLatencyWaitableObject operates - //DX::ThrowIfFailed(m_swapChain->SetMaximumFrameLatency(1)); - //m_frameLatencyWaitableObject = m_swapChain->GetFrameLatencyWaitableObject(); - // Associate swap chain with SwapChainPanel // UI changes will need to be dispatched back to the UI thread m_swapChainPanel->Dispatcher->RunAsync(CoreDispatcherPriority::High, ref new DispatchedHandler([=]() @@ -529,11 +507,6 @@ void DX::DeviceResources::SetDpi(float dpi) } } -void DX::DeviceResources::SetForceTearing(bool forceTearing) -{ - m_forceTearing = forceTearing; -} - // This method is called in the event handler for the OrientationChanged event. void DX::DeviceResources::SetCurrentOrientation(DisplayOrientations currentOrientation) { @@ -653,7 +626,7 @@ void DX::DeviceResources::Present() HRESULT hr = E_FAIL; DXGI_PRESENT_PARAMETERS parameters = { 0 }; - LOCK_D3D("Present"); + //LOCK_D3D("Present"); if (m_enableVsync) { // This will still use vsync due to the lack of DXGI_PRESENT_ALLOW_TEARING @@ -676,7 +649,7 @@ void DX::DeviceResources::Present() // Discard the contents of the depth stencil. m_d3dContext->DiscardView1(m_d3dDepthStencilView.Get(), nullptr, 0); - UNLOCK_D3D(); + //UNLOCK_D3D(); // If the device was removed either by a disconnection or a driver upgrade, we // must recreate all device resources. @@ -756,15 +729,6 @@ DXGI_MODE_ROTATION DX::DeviceResources::ComputeDisplayRotation() return rotation; } -// Block the current thread until the swap chain has finished presenting. -void DX::DeviceResources::WaitOnSwapChain() -{ - DWORD result = WaitForSingleObjectEx( - m_frameLatencyWaitableObject, - 1000, // 1 second timeout (shouldn't ever occur) - true); -} - //Thank you tunip3 for https://github.com/libretro/RetroArch/pull/13406/files //Check if we are running on Xbox bool is_running_on_xbox(void) diff --git a/Common/DeviceResources.h b/Common/DeviceResources.h index 686cdfd7..b5fead8d 100644 --- a/Common/DeviceResources.h +++ b/Common/DeviceResources.h @@ -30,7 +30,6 @@ namespace DX void WaitOnSwapChain(); void GetUWPPixelDimensions(uint32_t *width, uint32_t *height); double GetUWPRefreshRate(); - void SetForceTearing(bool forceTearing); static int uwp_get_width(); static int uwp_get_height(); @@ -86,7 +85,6 @@ namespace DX Microsoft::WRL::ComPtr m_d3dDevice; Microsoft::WRL::ComPtr m_d3dContext; Microsoft::WRL::ComPtr m_swapChain; - HANDLE m_frameLatencyWaitableObject; // Direct3D rendering objects. Required for 3D. Microsoft::WRL::ComPtr m_d3dRenderTargetView; @@ -127,7 +125,6 @@ namespace DX DXGI_FORMAT m_backBufferFormat; bool m_enableVsync; bool m_swapchainVsync; - bool m_forceTearing; bool m_showImGui; SyncMode m_displayStatus; std::shared_ptr m_stats; diff --git a/Pages/AppPage.xaml.cpp b/Pages/AppPage.xaml.cpp index f83a0b98..9d4ef7ab 100644 --- a/Pages/AppPage.xaml.cpp +++ b/Pages/AppPage.xaml.cpp @@ -74,7 +74,6 @@ void AppPage::Connect(int appId) { config->enableSOPS = host->EnableSOPS; config->enableStats = host->EnableStats; config->enableGraphs = host->EnableGraphs; - config->forceTearing = host->ForceTearing; if (config->enableHDR) { host->VideoCodec = "HEVC (H.265)"; } diff --git a/Pages/HostSettingsPage.xaml b/Pages/HostSettingsPage.xaml index a4e856fb..5b9e69f7 100644 --- a/Pages/HostSettingsPage.xaml +++ b/Pages/HostSettingsPage.xaml @@ -39,8 +39,6 @@ - - Resolution @@ -69,42 +67,27 @@ Enable HDR: - Optimize game settings: + Optimize host resolution: - Show performance stats: + Enable V-Sync: + + + Show performance stats: - Show performance graphs: + Show performance graphs: - Other: - - - - - - - - - - - The settings below are experimental, edit these at your own risk! - - - - - Force Tearing: - - Enable V-Sync: - + Other: + diff --git a/Pages/StreamPage.xaml.cpp b/Pages/StreamPage.xaml.cpp index 79e0ebff..c41a1878 100644 --- a/Pages/StreamPage.xaml.cpp +++ b/Pages/StreamPage.xaml.cpp @@ -142,7 +142,6 @@ void StreamPage::toggleLogsButton_Click(Platform::Object^ sender, Windows::UI::X void StreamPage::OnNavigatedTo(Windows::UI::Xaml::Navigation::NavigationEventArgs^ e) { configuration = dynamic_cast(e->Parameter); SetStreamConfig(configuration); - m_deviceResources->SetForceTearing(configuration->forceTearing); if (configuration == nullptr)return; } @@ -232,14 +231,14 @@ void StreamPage::toggleHDR_WinAltB_Click(Platform::Object^ sender, Windows::UI:: void StreamPage::increaseFrameDropTarget_Click(Platform::Object^ sender, Windows::UI::Xaml::RoutedEventArgs^ e) { - int target = 2; + int target = 1; target = FFMpegDecoder::instance().ModifyFrameDropTarget(true); // true increases the value this->frameDropTargetLabel->Text = Utils::StringPrintf("Frame queue size: %d", target); } void StreamPage::decreaseFrameDropTarget_Click(Platform::Object^ sender, Windows::UI::Xaml::RoutedEventArgs^ e) { - int target = 2; + int target = 1; target = FFMpegDecoder::instance().ModifyFrameDropTarget(false); // false decreases the value this->frameDropTargetLabel->Text = Utils::StringPrintf("Frame queue size: %d", target); } diff --git a/State/ApplicationState.cpp b/State/ApplicationState.cpp index 038f12cc..a34d7019 100644 --- a/State/ApplicationState.cpp +++ b/State/ApplicationState.cpp @@ -47,7 +47,6 @@ Concurrency::task moonlight_xbox_dx::ApplicationState::Init() if (a.contains("enable_sops")) h->EnableSOPS = a["enable_sops"].get(); if (a.contains("enable_stats")) h->EnableStats = a["enable_stats"].get(); if (a.contains("enable_graphs")) h->EnableGraphs = a["enable_graphs"].get(); - if (a.contains("force_tearing")) h->ForceTearing = a["force_tearing"].get(); if (a.contains("enable_vsync")) h->EnableVsync = a["enable_vsync"].get(); if (a.contains("serverAddress")) h->ServerAddress = Utils::StringFromStdString(a["serverAddress"].get()); if (a.contains("macaddress")) h->MacAddress = Utils::StringFromStdString(a["macaddress"].get()); @@ -108,7 +107,6 @@ Concurrency::task moonlight_xbox_dx::ApplicationState::UpdateFile() hostJson["enable_sops"] = host->EnableSOPS; hostJson["enable_stats"] = host->EnableStats; hostJson["enable_graphs"] = host->EnableGraphs; - hostJson["force_tearing"] = host->ForceTearing; hostJson["enable_vsync"] = host->EnableVsync; hostJson["serverAddress"] = Utils::PlatformStringToStdString(host->ServerAddress); diff --git a/State/MoonlightHost.h b/State/MoonlightHost.h index fbc5d043..f214e3af 100644 --- a/State/MoonlightHost.h +++ b/State/MoonlightHost.h @@ -30,7 +30,6 @@ namespace moonlight_xbox_dx { bool enableStats = false; bool enableGraphs = true; bool enableVsync = true; - bool forceTearing = false; Windows::Foundation::Collections::IVector^ apps; public: //Thanks to https://phsucharee.wordpress.com/2013/06/19/data-binding-and-ccx-inotifypropertychanged/ @@ -256,17 +255,6 @@ namespace moonlight_xbox_dx { } } - property bool ForceTearing - { - bool get() { return this->forceTearing; } - void set(bool value) { - this->forceTearing = value; - OnPropertyChanged("ForceTearing"); - } - } - - - // NOTE: vsync is currently forced ON until we can figure out how to get VRR working property bool EnableVsync { diff --git a/State/StreamConfiguration.h b/State/StreamConfiguration.h index 34aaace5..b027dc93 100644 --- a/State/StreamConfiguration.h +++ b/State/StreamConfiguration.h @@ -19,7 +19,6 @@ namespace moonlight_xbox_dx property bool enableSOPS; property bool enableStats; property bool enableGraphs; - property bool forceTearing; }; moonlight_xbox_dx::StreamConfiguration^ GetStreamConfig(); diff --git a/Streaming/FFmpegDecoder.cpp b/Streaming/FFmpegDecoder.cpp index 976663a6..0d03b0ba 100644 --- a/Streaming/FFmpegDecoder.cpp +++ b/Streaming/FFmpegDecoder.cpp @@ -55,7 +55,7 @@ namespace moonlight_xbox_dx { ffmpeg_buffer(nullptr), ffmpeg_buffer_size(0), resources(nullptr), - frameDropTarget(2), + m_FrameDropTarget(1), m_LastFrameNumber(0), m_Pacer(nullptr) { } @@ -97,7 +97,7 @@ namespace moonlight_xbox_dx { this->width = width; this->height = height; - this->frameDropTarget = 2; // user can modify this value + this->m_FrameDropTarget = 1; // user can modify this value this->m_LastFrameNumber = 0; this->ffmpeg_buffer_size = 0; @@ -175,6 +175,14 @@ namespace moonlight_xbox_dx { return -1; } + // Put D3D11 in multithread-friendly mode + ID3D11Multithread *pMultithread = nullptr; + HRESULT hr = d3d11va_device_ctx->device->QueryInterface(__uuidof(ID3D11Multithread), (void **)&pMultithread); + if (SUCCEEDED(hr)) { + pMultithread->SetMultithreadProtected(TRUE); + pMultithread->Release(); + } + GAMING_DEVICE_MODEL_INFORMATION info = {}; GetGamingDeviceModelInformation(&info); if (info.vendorId == GAMING_DEVICE_VENDOR_ID_MICROSOFT && @@ -201,15 +209,31 @@ namespace moonlight_xbox_dx { Utils::Log("FFMpegDecoder::Cleanup\n"); } - // Called by the VideoDec thread + static inline int frame_attach_userdata(AVFrame *frame, uint32_t presentationTimeMs, int64_t decodeEndQpc) { + if (!frame) return AVERROR(EINVAL); + + if (frame->opaque_ref) { + av_buffer_unref(&frame->opaque_ref); + } + + AVBufferRef *buf = av_buffer_allocz(sizeof(MLFrameData)); + if (!buf) return AVERROR(ENOMEM); + + MLFrameData *data = (MLFrameData *)buf->data; + data->presentationTimeMs = presentationTimeMs; + data->decodeEndQpc = decodeEndQpc; + frame->opaque_ref = buf; + + return 0; + } + + // Called by the VideoDec thread int FFMpegDecoder::SubmitDecodeUnit(PDECODE_UNIT decodeUnit) { LARGE_INTEGER decodeStart, decodeEnd; PLENTRY entry = decodeUnit->bufferList; int length = 0; QueryPerformanceCounter(&decodeStart); - FQLog("*** SubmitDecodeUnit(frame=%d, length=%d)\n", decodeUnit->frameNumber, decodeUnit->fullLength); - if (!ensure_buf_size(&ffmpeg_buffer, &ffmpeg_buffer_size, decodeUnit->fullLength + AV_INPUT_BUFFER_PADDING_SIZE)) { Utils::Logf("Couldn't realloc ffmpeg_buffer\n"); return DR_NEED_IDR; @@ -225,7 +249,7 @@ namespace moonlight_xbox_dx { // Detect breaks in the frame sequence indicating dropped packets uint32_t droppedFramesNetwork = 0; if (m_LastFrameNumber > 0 && decodeUnit->frameNumber > (m_LastFrameNumber + 1)) { - // Any frame number greater than m_LastFrameNumber + 1 represents a dropped fraFme + // Any frame number greater than m_LastFrameNumber + 1 represents a dropped frame droppedFramesNetwork = decodeUnit->frameNumber - (m_LastFrameNumber + 1); } m_LastFrameNumber = decodeUnit->frameNumber; @@ -248,13 +272,11 @@ namespace moonlight_xbox_dx { Utils::Logf("avcodec_send_packet failed: %s\n", ffmpegError); return DR_NEED_IDR; } - FQLog("avcodec_send_packet size %d\n", length); while (err >= 0) { AVFrame* frame = av_frame_alloc(); err = avcodec_receive_frame(decoder_ctx, frame); if (err == AVERROR(EAGAIN) || err == AVERROR_EOF) { - FQLog("avcodec_receive_frame returned %d\n", err); av_frame_free(&frame); break; } @@ -268,20 +290,17 @@ namespace moonlight_xbox_dx { // Capture a frame timestamp to measuring pacing delay QueryPerformanceCounter(&decodeEnd); - //frame->pkt_dts = decodeEnd.QuadPart; - - // Store the presentation time - //frame->pts = decodeUnit->presentationTimeMs; + frame_attach_userdata(frame, decodeUnit->presentationTimeMs, decodeEnd.QuadPart); // Queue the frame for rendering (or render now if pacer is disabled) m_Pacer->submitFrame(frame); double decodeTimeMs = QpcToMs(decodeEnd.QuadPart - decodeStart.QuadPart); - FQLog("avcodec_receive_frame got frame %3"PRId64" in %.3f ms (pts %llu)\n", decoder_ctx->frame_num, decodeTimeMs, frame->pts); - // if (decodeUnit->frameNumber > 200 && decodeTimeMs > 12.0f) { - // __debugbreak; - // assert(false); - // } + FQLog("✓ Frame decoded [pts: %.3f] [in#: %d] [out#: %d] [lost: %d] %.3f ms\n", + (frame->pts / 90000.0) * 1000.0, + decodeUnit->frameNumber, decoder_ctx->frame_num, + decoder_ctx->frame_num - decodeUnit->frameNumber, + QpcToMs(decodeEnd.QuadPart - decodeStart.QuadPart)); // Even though we have a valid frame, the ffmpeg API needs us to loop and call avcodec_receive_frame() // again where we expect to get AVERROR(EAGAIN) and break out. @@ -300,20 +319,7 @@ namespace moonlight_xbox_dx { } int FFMpegDecoder::ModifyFrameDropTarget(bool increase) { - int current = frameDropTarget; - if (increase) { - if (current >= 14) { - // decodeUnitQueue is 15, so our max here is 14 - return current; - } - return ++frameDropTarget; - } - else { - if (current == 0) { - return current; - } - return --frameDropTarget; - } + return m_Pacer->modifyFrameDropTarget(increase); } //Helpers diff --git a/Streaming/FFmpegDecoder.h b/Streaming/FFmpegDecoder.h index 7e2e3635..a0c5cec8 100644 --- a/Streaming/FFmpegDecoder.h +++ b/Streaming/FFmpegDecoder.h @@ -4,68 +4,71 @@ #include #include "../Common/StepTimer.h" #include "Pacer.h" -#include "VideoRenderer.h" #include "Utils.hpp" +#include "VideoRenderer.h" extern "C" { - #include - #include - #include - #include +#include +#include +#include +#include } #define MAX_BUFFER 1024 * 1024 -namespace moonlight_xbox_dx -{ +typedef struct MLFrameData { + uint32_t presentationTimeMs; + int64_t decodeEndQpc; +} MLFrameData; - class FFMpegDecoder { - public: - // Singleton accessor - static FFMpegDecoder &instance(); +namespace moonlight_xbox_dx { - void CompleteInitialization(const std::shared_ptr& res, STREAM_CONFIGURATION *config); - int Init(int videoFormat, int width, int height, int redrawRate, void* context, int drFlags); - void Cleanup(); - int SubmitDecodeUnit(PDECODE_UNIT decodeUnit); - int ModifyFrameDropTarget(bool increase); - void WaitForFrame(); - bool RenderFrameOnMainThread(std::shared_ptr& sceneRenderer); - static FFMpegDecoder* getInstance(); - static DECODER_RENDERER_CALLBACKS getDecoder(); - std::recursive_mutex mutex; - bool hackWait = false; - int videoFormat, width, height; +class FFMpegDecoder { + public: + // Singleton accessor + static FFMpegDecoder &instance(); - private: - FFMpegDecoder(); - FFMpegDecoder(const FFMpegDecoder &) = delete; - FFMpegDecoder &operator=(const FFMpegDecoder &) = delete; + void CompleteInitialization(const std::shared_ptr &res, STREAM_CONFIGURATION *config); + int Init(int videoFormat, int width, int height, int redrawRate, void *context, int drFlags); + void Cleanup(); + int SubmitDecodeUnit(PDECODE_UNIT decodeUnit); + int ModifyFrameDropTarget(bool increase); + void WaitForFrame(); + bool RenderFrameOnMainThread(std::shared_ptr &sceneRenderer); + static FFMpegDecoder *getInstance(); + static DECODER_RENDERER_CALLBACKS getDecoder(); + std::recursive_mutex mutex; + bool hackWait = false; + int videoFormat, width, height; - const AVCodec* decoder; - AVCodecContext* decoder_ctx; - AVHWDeviceContext* device_ctx; - AVD3D11VADeviceContext* d3d11va_device_ctx; - unsigned char* ffmpeg_buffer; - int ffmpeg_buffer_size; - std::shared_ptr resources; - std::atomic_int frameDropTarget; - int m_LastFrameNumber; - std::unique_ptr m_Pacer; - }; -} + private: + FFMpegDecoder(); + FFMpegDecoder(const FFMpegDecoder &) = delete; + FFMpegDecoder &operator=(const FFMpegDecoder &) = delete; + + const AVCodec *decoder; + AVCodecContext *decoder_ctx; + AVHWDeviceContext *device_ctx; + AVD3D11VADeviceContext *d3d11va_device_ctx; + unsigned char *ffmpeg_buffer; + int ffmpeg_buffer_size; + std::shared_ptr resources; + int m_LastFrameNumber; + std::unique_ptr m_Pacer; +}; +} // namespace moonlight_xbox_dx inline void LOCK_D3D(const char *msg) { -#if !defined(NDEBUG) +#ifdef FRAME_QUEUE_VERBOSE LARGE_INTEGER t0, t1; QueryPerformanceCounter(&t0); #endif FFMpegDecoder::instance().mutex.lock(); -#if !defined(NDEBUG) +#ifdef FRAME_QUEUE_VERBOSE QueryPerformanceCounter(&t1); - Utils::Logf("%s acquired lock_context in %.3f ms\n", msg, QpcToMs(t1.QuadPart - t0.QuadPart)); + //FQLog("%s acquired lock_context in %.3f ms\n", msg, QpcToMs(t1.QuadPart - t0.QuadPart)); #endif } diff --git a/Streaming/Pacer.cpp b/Streaming/Pacer.cpp index e792152d..c00124b1 100644 --- a/Streaming/Pacer.cpp +++ b/Streaming/Pacer.cpp @@ -1,5 +1,6 @@ #include "pch.h" #include "Pacer.h" +#include "FFmpegDecoder.h" #include "../Plot/ImGuiPlots.h" #include "Utils.hpp" @@ -14,7 +15,8 @@ Pacer::Pacer(const std::shared_ptr& res) : m_DeviceResources(res), m_Stopping(false), m_MaxVideoFps(0), - m_DisplayFps(0.0f) + m_DisplayFps(0.0f), + m_FrameDropTarget(1) { } @@ -49,9 +51,6 @@ bool Pacer::renderOnMainThread(std::shared_ptr& sceneRenderer) if (!m_RenderQueue.empty()) { frame = std::move(m_RenderQueue.front()); m_RenderQueue.pop_front(); - - FQLog("[<- pts %llu] dequeue frame, queue size %d\n", - frame->pts, m_RenderQueue.size()); } } @@ -82,21 +81,26 @@ void Pacer::renderFrame(std::shared_ptr& sceneRenderer, AVFrame* sceneRenderer->Render(frame); QueryPerformanceCounter(&afterRender); - m_DeviceResources->GetStats()->SubmitPacerTime( - beforeRender.QuadPart - frame->pkt_dts, // pkt_dts is afterDecode.QuadPart - afterRender.QuadPart - beforeRender.QuadPart); + if (frame->opaque_ref) { + MLFrameData *data = (MLFrameData *)frame->opaque_ref->data; + + m_DeviceResources->GetStats()->SubmitPacerTime( + beforeRender.QuadPart - data->decodeEndQpc, + afterRender.QuadPart - beforeRender.QuadPart); + } av_frame_free(&frame); // Drop frames if we have too many queued up for a while m_FrameQueueLock.lock(); - int frameDropTarget = 0; + // these were originally 0 and 2 + int frameDropTarget = m_FrameDropTarget; for (int queueHistoryEntry : m_RenderQueueHistory) { if (queueHistoryEntry == 0) { // Be lenient as long as the queue length // resolves before the end of frame history - frameDropTarget = 2; + frameDropTarget += 2; break; } } @@ -115,8 +119,8 @@ void Pacer::renderFrame(std::shared_ptr& sceneRenderer, AVFrame* AVFrame* frame = std::move(m_RenderQueue.front()); m_RenderQueue.pop_front(); - Utils::Logf("pacer queue %d > frame drop target %d, dropping oldest frame (pts %lld)\n", - m_RenderQueue.size() + 1, frameDropTarget, frame->pts); + FQLog("!! pacer queue overflow: %d > frame drop target %d, dropping oldest frame (pts %.3f ms)\n", + m_RenderQueue.size() + 1, frameDropTarget, (frame->pts / 90000.0) * 1000); m_DeviceResources->GetStats()->SubmitDroppedFrame(1); // Drop the lock while we call av_frame_free() @@ -142,8 +146,8 @@ void Pacer::dropFrameForEnqueue(std::deque& queue) AVFrame* frame = std::move(queue.front()); queue.pop_front(); - Utils::Logf("pacer queue full (%d), dropping oldest frame (pts %lld)\n", - queue.size() + 1, frame->pts); + FQLog("!! pacer queue full (%d), dropping oldest frame (pts %.3f ms)\n", + queue.size() + 1, (frame->pts / 90000.0) * 1000); m_DeviceResources->GetStats()->SubmitDroppedFrame(1); av_frame_free(&frame); @@ -166,9 +170,24 @@ void Pacer::submitFrame(AVFrame* frame) size_t queueSize = m_RenderQueue.size(); ImGuiPlots::instance().observeFloat(PLOT_QUEUED_FRAMES, (float)queueSize); - FQLog("[-> pts %lld] enqueue frame, render queue size %d\n", frame->pts, queueSize); } // notify render loop of new frame m_RenderQueueNotEmpty.notify_one(); } + +int Pacer::modifyFrameDropTarget(bool increase) { + int current = m_FrameDropTarget; + if (increase) { + if (current >= 5) { + return current; + } + return ++m_FrameDropTarget; + } + else { + if (current == 0) { + return current; + } + return --m_FrameDropTarget; + } +} diff --git a/Streaming/Pacer.h b/Streaming/Pacer.h index d192b491..343a4b66 100644 --- a/Streaming/Pacer.h +++ b/Streaming/Pacer.h @@ -26,6 +26,8 @@ class Pacer bool renderOnMainThread(std::shared_ptr& sceneRenderer); + int modifyFrameDropTarget(bool increase); + private: void renderFrame(std::shared_ptr& sceneRenderer, AVFrame* frame); @@ -39,4 +41,5 @@ class Pacer std::atomic m_Stopping{false}; int m_MaxVideoFps; double m_DisplayFps; + std::atomic_int m_FrameDropTarget; }; diff --git a/Streaming/VideoRenderer.cpp b/Streaming/VideoRenderer.cpp index e18ff6ce..490d9d6c 100644 --- a/Streaming/VideoRenderer.cpp +++ b/Streaming/VideoRenderer.cpp @@ -109,7 +109,7 @@ bool VideoRenderer::Render(AVFrame* frame) return true; } - FQLog("[%lld ms] rendering frame\n", frame->pts); + //LOCK_D3D("VideoRenderer->Render"); auto d3dcontext = m_deviceResources->GetD3DDeviceContext(); auto d3ddevice = m_deviceResources->GetD3DDevice(); @@ -203,6 +203,8 @@ bool VideoRenderer::Render(AVFrame* frame) m_LastColorTrc = frame->color_trc; } + //UNLOCK_D3D(); + return true; } diff --git a/Streaming/moonlight_xbox_dxMain.cpp b/Streaming/moonlight_xbox_dxMain.cpp index d748a8b7..8a8477b7 100644 --- a/Streaming/moonlight_xbox_dxMain.cpp +++ b/Streaming/moonlight_xbox_dxMain.cpp @@ -390,33 +390,33 @@ bool moonlight_xbox_dxMain::Render() return false; } - LOCK_D3D("Clear"); + //LOCK_D3D("Clear"); Clear(); - UNLOCK_D3D(); + //UNLOCK_D3D(); // Start the Dear ImGui frame, limited to only running at 60fps bool showImGui = m_deviceResources->GetShowImGui(); if (showImGui) { - LOCK_D3D("ImGui 1"); + //LOCK_D3D("ImGui 1"); ImGui_ImplDX11_NewFrame(); - UNLOCK_D3D(); + //UNLOCK_D3D(); ImGui_ImplUwp_NewFrame(m_deviceResources->GetPixelWidth(), m_deviceResources->GetPixelHeight()); ImGui::NewFrame(); } // Render the scene objects. - LOCK_D3D("Render"); + //LOCK_D3D("Render"); bool shouldPresent = FFMpegDecoder::instance().RenderFrameOnMainThread(m_sceneRenderer); m_LogRenderer->Render(); m_statsTextRenderer->Render(showImGui); - UNLOCK_D3D(); + //UNLOCK_D3D(); if (showImGui) { RenderImGui(); ImGui::Render(); - LOCK_D3D("ImGui 2"); + //LOCK_D3D("ImGui 2"); ImGui_ImplDX11_RenderDrawData(ImGui::GetDrawData()); - UNLOCK_D3D(); + //UNLOCK_D3D(); } return shouldPresent; diff --git a/pch.h b/pch.h index 9d62c590..1b6cddb4 100644 --- a/pch.h +++ b/pch.h @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include From 84c31a9fd1ee5f143bf4d96fc9203a24ee6b4b95 Mon Sep 17 00:00:00 2001 From: Andy Grundman Date: Thu, 18 Sep 2025 13:37:35 -0400 Subject: [PATCH 07/22] Point moonlight-common-c to repo with Xbox patches --- .gitmodules | 2 +- third_party/moonlight-common-c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 335a843a..870562b1 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,6 @@ [submodule "third_party/moonlight-common-c"] path = third_party/moonlight-common-c - url = https://github.com/moonlight-stream/moonlight-common-c.git + url = https://github.com/andygrundman/moonlight-common-c.git [submodule "vcpkg"] path = vcpkg url = https://github.com/microsoft/vcpkg.git diff --git a/third_party/moonlight-common-c b/third_party/moonlight-common-c index a56638e8..4372ab39 160000 --- a/third_party/moonlight-common-c +++ b/third_party/moonlight-common-c @@ -1 +1 @@ -Subproject commit a56638e86c9197f3e626f2beefad079b11df79f3 +Subproject commit 4372ab392c3532adabc3b087180d79cd011b762f From e0e32a23636c2ff0a72b316df1989d1e572d8c7b Mon Sep 17 00:00:00 2001 From: Andy Grundman Date: Thu, 18 Sep 2025 13:53:50 -0400 Subject: [PATCH 08/22] Full port of qt's Pacer class. waitForVsync does not appear to work correctly with Xbox so it's commented out. --- Common/DeviceResources.cpp | 13 +- Common/DeviceResources.h | 4 + Plot/ImGuiPlots.cpp | 3 +- Plot/PlotDesc.h | 6 +- State/Stats.cpp | 17 +- State/Stats.h | 2 +- Streaming/FFmpegDecoder.cpp | 6 +- Streaming/FFmpegDecoder.h | 4 +- Streaming/Pacer.cpp | 236 ++++++++++++++++++++++++---- Streaming/Pacer.h | 24 ++- Streaming/StatsRenderer.cpp | 11 +- Streaming/moonlight_xbox_dxMain.cpp | 12 +- moonlight-xbox-dx.vcxproj | 1 + moonlight-xbox-dx.vcxproj.filters | 224 ++++++++++++++++++++++++++ pch.h | 35 ++--- 15 files changed, 504 insertions(+), 94 deletions(-) diff --git a/Common/DeviceResources.cpp b/Common/DeviceResources.cpp index 428adf66..4fec2756 100644 --- a/Common/DeviceResources.cpp +++ b/Common/DeviceResources.cpp @@ -243,12 +243,14 @@ void DX::DeviceResources::CreateWindowSizeDependentResources() if (m_swapChain != nullptr) { // If the swap chain already exists, resize it. + int flags = DXGI_SWAP_CHAIN_FLAG_FRAME_LATENCY_WAITABLE_OBJECT; + if (!m_enableVsync) flags |= DXGI_SWAP_CHAIN_FLAG_ALLOW_TEARING; HRESULT hr = m_swapChain->ResizeBuffers( 2, // Double-buffered swap chain. lround(m_d3dRenderTargetSize.Width), lround(m_d3dRenderTargetSize.Height), m_backBufferFormat, - m_enableVsync ? 0 : DXGI_SWAP_CHAIN_FLAG_ALLOW_TEARING + flags ); if (hr == DXGI_ERROR_DEVICE_REMOVED || hr == DXGI_ERROR_DEVICE_RESET) @@ -280,7 +282,8 @@ void DX::DeviceResources::CreateWindowSizeDependentResources() //Check moonlight-stream/moonlight-qt/app/streaming/video/ffmpeg-renderers/d3d11va.cpp for rationale swapChainDesc.BufferCount = 5; swapChainDesc.SwapEffect = DXGI_SWAP_EFFECT_FLIP_DISCARD; - swapChainDesc.Flags = m_enableVsync ? 0 : DXGI_SWAP_CHAIN_FLAG_ALLOW_TEARING; + swapChainDesc.Flags = DXGI_SWAP_CHAIN_FLAG_FRAME_LATENCY_WAITABLE_OBJECT; + if (!m_enableVsync) swapChainDesc.Flags |= DXGI_SWAP_CHAIN_FLAG_ALLOW_TEARING; swapChainDesc.Scaling = DXGI_SCALING_STRETCH; swapChainDesc.AlphaMode = DXGI_ALPHA_MODE_IGNORE; @@ -295,6 +298,9 @@ void DX::DeviceResources::CreateWindowSizeDependentResources() dxgiDevice->GetAdapter(&dxgiAdapter) ); + DX::ThrowIfFailed( + dxgiAdapter->EnumOutputs(0, &m_dxgiOutput)); + ComPtr dxgiFactory; DX::ThrowIfFailed( dxgiAdapter->GetParent(IID_PPV_ARGS(&dxgiFactory)) @@ -317,6 +323,9 @@ void DX::DeviceResources::CreateWindowSizeDependentResources() swapChain.As(&m_swapChain) ); + //m_swapChain->SetMaximumFrameLatency(1); + m_frameLatencyWaitable = m_swapChain->GetFrameLatencyWaitableObject(); + // Associate swap chain with SwapChainPanel // UI changes will need to be dispatched back to the UI thread m_swapChainPanel->Dispatcher->RunAsync(CoreDispatcherPriority::High, ref new DispatchedHandler([=]() diff --git a/Common/DeviceResources.h b/Common/DeviceResources.h index b5fead8d..ba541def 100644 --- a/Common/DeviceResources.h +++ b/Common/DeviceResources.h @@ -60,6 +60,8 @@ namespace DX ID3D11Device3* GetD3DDevice() const { return m_d3dDevice.Get(); } ID3D11DeviceContext3* GetD3DDeviceContext() const { return m_d3dContext.Get(); } IDXGISwapChain4* GetSwapChain() const { return m_swapChain.Get(); } + IDXGIOutput* GetDXGIOutput() const { return m_dxgiOutput.Get(); } + HANDLE GetFrameLatencyWaitable() const { return m_frameLatencyWaitable; } auto GetDXGIFactory() const noexcept { return m_dxgiFactory.Get(); } D3D_FEATURE_LEVEL GetDeviceFeatureLevel() const { return m_d3dFeatureLevel; } ID3D11RenderTargetView1* GetBackBufferRenderTargetView() const { return m_d3dRenderTargetView.Get(); } @@ -85,6 +87,8 @@ namespace DX Microsoft::WRL::ComPtr m_d3dDevice; Microsoft::WRL::ComPtr m_d3dContext; Microsoft::WRL::ComPtr m_swapChain; + Microsoft::WRL::ComPtr m_dxgiOutput; + HANDLE m_frameLatencyWaitable; // Direct3D rendering objects. Required for 3D. Microsoft::WRL::ComPtr m_d3dRenderTargetView; diff --git a/Plot/ImGuiPlots.cpp b/Plot/ImGuiPlots.cpp index 7348d603..96a99745 100644 --- a/Plot/ImGuiPlots.cpp +++ b/Plot/ImGuiPlots.cpp @@ -23,7 +23,8 @@ ImGuiPlots::ImGuiPlots() : Plot(kPlotDescs[PLOT_HOST_FRAMETIME]), Plot(kPlotDescs[PLOT_QUEUED_FRAMES]), Plot(kPlotDescs[PLOT_DROPPED_NETWORK]), - Plot(kPlotDescs[PLOT_DROPPED_PACER]), + Plot(kPlotDescs[PLOT_DROPPED_PACER_BACK]), + Plot(kPlotDescs[PLOT_DROPPED_PACER_FRONT]), Plot(kPlotDescs[PLOT_OVERHEAD]), Plot(kPlotDescs[PLOT_ETC]), diff --git a/Plot/PlotDesc.h b/Plot/PlotDesc.h index 313707cf..240ac4f7 100644 --- a/Plot/PlotDesc.h +++ b/Plot/PlotDesc.h @@ -9,7 +9,8 @@ enum PlotType PLOT_HOST_FRAMETIME, PLOT_QUEUED_FRAMES, PLOT_DROPPED_NETWORK, - PLOT_DROPPED_PACER, + PLOT_DROPPED_PACER_BACK, + PLOT_DROPPED_PACER_FRONT, PLOT_OVERHEAD, PLOT_ETC, PlotCount @@ -39,7 +40,8 @@ inline constexpr std::array kPlotDescs = {{ {"Host Frametime", PLOT_LABEL_MIN_MAX_AVG, "ms", -0.1f, 50.0f, 0.0f, 49.9f}, {"Queued Frames", PLOT_LABEL_MIN_MAX_AVG, "", -1.0f, 10.0f, 0.0f, 0.0f}, {"Dropped Frames (network)", PLOT_LABEL_TOTAL_INT, "", -1.0f, 3.0f, 0.0f, 0.0f}, - {"Dropped Frames (pacing queue)", PLOT_LABEL_TOTAL_INT, "", -1.0f, 3.0f, 0.0f, 0.0f}, + {"Dropped Frames (back pacing)", PLOT_LABEL_TOTAL_INT, "", -1.0f, 3.0f, 0.0f, 0.0f}, + {"Dropped Frames (front pacing)", PLOT_LABEL_TOTAL_INT, "", -1.0f, 3.0f, 0.0f, 0.0f}, {"Graph overhead", PLOT_LABEL_MIN_MAX_AVG, "ms", -0.1f, 6.0f, 0.0f, 0.0f}, {"Etc...", PLOT_LABEL_MIN_MAX_AVG, "ms", -0.1f, 50.0f, 0.0f, 49.9f}, }}; diff --git a/State/Stats.cpp b/State/Stats.cpp index a6c101fb..23f16278 100644 --- a/State/Stats.cpp +++ b/State/Stats.cpp @@ -91,15 +91,12 @@ void Stats::SubmitVideoBytesAndReassemblyTime(uint32_t length, PDECODE_UNIT deco } ImGuiPlots::instance().observeFloat(PLOT_DROPPED_NETWORK, droppedFrames); - // Host frametime graph - static unsigned int lastHostPts = 0; + // Host frametime graph, uses raw 90kHz units to avoid rounding errors + static uint32_t lastHostPts = 0; if (lastHostPts > 0) { - ImGuiPlots::instance().observeFloat(PLOT_HOST_FRAMETIME, (float)(decodeUnit->presentationTimeMs - lastHostPts)); + ImGuiPlots::instance().observeFloat(PLOT_HOST_FRAMETIME, (float)((decodeUnit->rtpTimestamp - lastHostPts) / 90.0f)); } - lastHostPts = decodeUnit->presentationTimeMs; - - // Queued frames for graph - //ImGuiPlots::instance().observeFloat(PLOT_QUEUED_FRAMES, LiGetPendingVideoFrames()); + lastHostPts = decodeUnit->rtpTimestamp; } // Time in milliseconds we spent decoding one frame, it is added up to later be divided by decodedFrames @@ -117,12 +114,12 @@ void Stats::SubmitDroppedFrame(int count) { // Time in microseconds we spent in the frame pacer, and time for rendering the frame. // Also increments the rendered frame count. -void Stats::SubmitPacerTime(uint64_t pacerTimeQpc, uint64_t renderTimeQpc) { +void Stats::SubmitPacerTime(int64_t pacerTimeQpc, int64_t renderTimeQpc) { std::lock_guard lock(m_mutex); - uint64_t pacerTimeUs = QpcToUs(pacerTimeQpc); + int64_t pacerTimeUs = QpcToUs(pacerTimeQpc); m_ActiveWndVideoStats.totalPacerTimeUs += pacerTimeUs; - uint64_t renderTimeUs = QpcToUs(renderTimeQpc); + int64_t renderTimeUs = QpcToUs(renderTimeQpc); m_ActiveWndVideoStats.totalRenderTimeUs += renderTimeUs; m_ActiveWndVideoStats.renderedFrames++; } diff --git a/State/Stats.h b/State/Stats.h index fabfca33..571ad90a 100644 --- a/State/Stats.h +++ b/State/Stats.h @@ -55,7 +55,7 @@ namespace moonlight_xbox_dx void SubmitVideoBytesAndReassemblyTime(uint32_t length, PDECODE_UNIT decodeUnit, uint32_t droppedFrames); void SubmitDecodeMs(double decodeMs); void SubmitDroppedFrame(int count); - void SubmitPacerTime(uint64_t pacerTimeQpc, uint64_t renderTimeQpc); + void SubmitPacerTime(int64_t pacerTimeQpc, int64_t renderTimeQpc); void SetDisplayStatus(SyncMode status) { m_displayStatus = status; } diff --git a/Streaming/FFmpegDecoder.cpp b/Streaming/FFmpegDecoder.cpp index 0d03b0ba..d235b7e1 100644 --- a/Streaming/FFmpegDecoder.cpp +++ b/Streaming/FFmpegDecoder.cpp @@ -55,7 +55,6 @@ namespace moonlight_xbox_dx { ffmpeg_buffer(nullptr), ffmpeg_buffer_size(0), resources(nullptr), - m_FrameDropTarget(1), m_LastFrameNumber(0), m_Pacer(nullptr) { } @@ -97,7 +96,6 @@ namespace moonlight_xbox_dx { this->width = width; this->height = height; - this->m_FrameDropTarget = 1; // user can modify this value this->m_LastFrameNumber = 0; this->ffmpeg_buffer_size = 0; @@ -296,8 +294,8 @@ namespace moonlight_xbox_dx { m_Pacer->submitFrame(frame); double decodeTimeMs = QpcToMs(decodeEnd.QuadPart - decodeStart.QuadPart); - FQLog("✓ Frame decoded [pts: %.3f] [in#: %d] [out#: %d] [lost: %d] %.3f ms\n", - (frame->pts / 90000.0) * 1000.0, + FQLog("✓ Frame decoded [pts: %.3f] [in#: %d] [out#: %d] [lost: %d] decode time %.3f ms\n", + frame->pts / 90.0, decodeUnit->frameNumber, decoder_ctx->frame_num, decoder_ctx->frame_num - decodeUnit->frameNumber, QpcToMs(decodeEnd.QuadPart - decodeStart.QuadPart)); diff --git a/Streaming/FFmpegDecoder.h b/Streaming/FFmpegDecoder.h index a0c5cec8..286e81a9 100644 --- a/Streaming/FFmpegDecoder.h +++ b/Streaming/FFmpegDecoder.h @@ -64,7 +64,7 @@ inline void LOCK_D3D(const char *msg) { QueryPerformanceCounter(&t0); #endif - FFMpegDecoder::instance().mutex.lock(); + moonlight_xbox_dx::FFMpegDecoder::instance().mutex.lock(); #ifdef FRAME_QUEUE_VERBOSE QueryPerformanceCounter(&t1); @@ -73,5 +73,5 @@ inline void LOCK_D3D(const char *msg) { } inline void UNLOCK_D3D() { - FFMpegDecoder::instance().mutex.unlock(); + moonlight_xbox_dx::FFMpegDecoder::instance().mutex.unlock(); } diff --git a/Streaming/Pacer.cpp b/Streaming/Pacer.cpp index c00124b1..1265bd0e 100644 --- a/Streaming/Pacer.cpp +++ b/Streaming/Pacer.cpp @@ -3,6 +3,9 @@ #include "FFmpegDecoder.h" #include "../Plot/ImGuiPlots.h" #include "Utils.hpp" +#include +#include +#include // Limit the number of queued frames to prevent excessive memory consumption // if the V-Sync source or renderer is blocked for a while. It's important @@ -11,12 +14,23 @@ // out of available decoding surfaces. #define MAX_QUEUED_FRAMES 4 +// We may be woken up slightly late so don't go all the way +// up to the next V-sync since we may accidentally step into +// the next V-sync period. It also takes some amount of time +// to do the render itself, so we can't render right before +// V-sync happens. +#define TIMER_SLACK_MS 3 + +using steady_clock = std::chrono::steady_clock; +using namespace moonlight_xbox_dx; + Pacer::Pacer(const std::shared_ptr& res) : m_DeviceResources(res), + m_VsyncThread(), m_Stopping(false), m_MaxVideoFps(0), - m_DisplayFps(0.0f), - m_FrameDropTarget(1) + m_DisplayFps(0.0), + m_FrameDropTarget(0) { } @@ -26,14 +40,170 @@ Pacer::~Pacer() m_Stopping = true; m_RenderQueueNotEmpty.notify_all(); - // Delete any remaining unconsumed frames - while (!m_RenderQueue.empty()) { - AVFrame* frame = std::move(m_RenderQueue.front()); - m_RenderQueue.pop_front(); + // Stop the V-sync thread + m_PacingQueueNotEmpty.notify_all(); + if (m_VsyncThread.joinable()) { + m_VsyncThread.join(); + } + + // Drain and free any remaining frames in both queues + { + std::unique_lock lock(m_FrameQueueLock); + + while (!m_RenderQueue.empty()) { + AVFrame* f = m_RenderQueue.front(); + m_RenderQueue.pop_front(); + lock.unlock(); + av_frame_free(&f); + lock.lock(); + } + while (!m_PacingQueue.empty()) { + AVFrame* f = m_PacingQueue.front(); + m_PacingQueue.pop_front(); + lock.unlock(); + av_frame_free(&f); + lock.lock(); + } + } +} + +void Pacer::initialize(int maxVideoFps, double refreshRate) +{ + m_MaxVideoFps = maxVideoFps; + m_DisplayFps = refreshRate; + + Utils::Logf("Frame pacing: target %.2f Hz with %d FPS stream", + m_DisplayFps, m_MaxVideoFps); + + // Start the dedicated vblank pacing thread once + if (!m_VsyncThread.joinable()) { + m_VsyncThread = std::thread(&Pacer::vsyncThread, this); + } +} + +// Vsync thread + +void Pacer::vsyncThread() +{ + if (!SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_TIME_CRITICAL)) { + Utils::Logf("Failed to set vsyncThread priority: %d\n", GetLastError()); + } + + while (!m_Stopping) { + // See comments in waitForVsync() + + LARGE_INTEGER beforeWait, afterWait; + // QueryPerformanceCounter(&beforeWait); + // waitForVsync(); + // QueryPerformanceCounter(&afterWait); + // FQLog("waitForVsync() waited %.3f ms\n", QpcToMs(afterWait.QuadPart - beforeWait.QuadPart)); + + // if (m_Stopping) { + // break; + // } + + QueryPerformanceCounter(&beforeWait); + handleVsync(1000.0 / m_DisplayFps); + QueryPerformanceCounter(&afterWait); + FQLog("handleVsync() waited %.3f ms\n", QpcToMs(afterWait.QuadPart - beforeWait.QuadPart)); + } +} + +void Pacer::waitForVsync() +{ + // XXX for some reason, both of these vsync methods seem to run at 60hz even when Xbox is running at 120hz + // We can probably just run without them, but perfect smoothness probably requires some vsync support + + // might work but is tied to a single display + //m_DeviceResources->GetDXGIOutput()->WaitForVBlank(); + + //HANDLE flw = m_DeviceResources->GetFrameLatencyWaitable(); + //WaitForSingleObject(flw, 2000 /*ms timeout*/); +} + +// Called in an arbitrary thread by the IVsyncSource on V-sync +// or an event synchronized with V-sync +void Pacer::handleVsync(double timeUntilNextVsyncMillis) +{ + // Make sure initialize() has been called + assert(m_MaxVideoFps != 0); + + std::unique_lock lock(m_FrameQueueLock); + + // If the queue length history entries are large, be strict + // about dropping excess frames. + int frameDropTarget = 1; + + // If we may get more frames per second than we can display, use + // frame history to drop frames only if consistently above the + // one queued frame mark. + if (m_MaxVideoFps >= m_DisplayFps) { + for (int queueHistoryEntry : m_PacingQueueHistory) { + if (queueHistoryEntry <= 1) { + // Be lenient as long as the queue length + // resolves before the end of frame history + frameDropTarget = 3; + break; + } + } + + // Keep a rolling 500 ms window of pacing queue history + if (m_PacingQueueHistory.size() == m_DisplayFps / 2) { + m_PacingQueueHistory.pop_front(); + } + + m_PacingQueueHistory.push_back((int)m_PacingQueue.size()); + } + + // Catch up if we're several frames ahead + int dropCount = 0; + while (m_PacingQueue.size() > frameDropTarget) { + ++dropCount; + AVFrame* frame = std::move(m_PacingQueue.front()); + m_PacingQueue.pop_front(); + + FQLog("! pacing queue overflow: %d > frame drop target %d, dropping oldest frame (pts %.3f ms)\n", + m_PacingQueue.size() + 1, frameDropTarget, frame->pts / 90.0); + m_DeviceResources->GetStats()->SubmitDroppedFrame(1); + + // Drop the lock while we call av_frame_free() + lock.unlock(); av_frame_free(&frame); + lock.lock(); + } + + // Don't graph 0 here, because it's already being done in dropFrameForEnqueue + if (dropCount > 0) { + ImGuiPlots::instance().observeFloat(PLOT_DROPPED_PACER_BACK, (float)dropCount); } + + if (m_PacingQueue.empty()) { + // Sleep up to the next vblank interval or until a frame arrives + static constexpr auto kSlack = std::chrono::milliseconds(TIMER_SLACK_MS); + const auto period_us = std::chrono::microseconds(static_cast(timeUntilNextVsyncMillis * 1000.0 + 0.5)); + const auto remaining = (period_us > kSlack) + ? (period_us - kSlack) + : std::chrono::microseconds(0); + + m_PacingQueueNotEmpty.wait_for(lock, remaining, [this]{ + return m_Stopping || !m_PacingQueue.empty(); + }); + + if (m_Stopping || m_PacingQueue.empty()) { + return; + } + } + + // Place the first frame on the render queue + AVFrame* frame = std::move(m_PacingQueue.front()); + m_PacingQueue.pop_front(); + lock.unlock(); + + enqueueFrameForRenderingAndUnlock(frame); } +// Main thread + void Pacer::waitForFrame() { std::unique_lock lock(m_FrameQueueLock); @@ -62,13 +232,19 @@ bool Pacer::renderOnMainThread(std::shared_ptr& sceneRenderer) return true; // ok to Present() } -void Pacer::initialize(int maxVideoFps, double refreshRate) +void Pacer::enqueueFrameForRenderingAndUnlock(AVFrame *frame) { - m_MaxVideoFps = maxVideoFps; - m_DisplayFps = refreshRate; + { + std::scoped_lock lock(m_FrameQueueLock); + dropFrameForEnqueue(m_RenderQueue, PLOT_DROPPED_PACER_FRONT); + m_RenderQueue.push_back(frame); - Utils::Logf("Frame pacing: target %.2f Hz with %d FPS stream", - m_DisplayFps, m_MaxVideoFps); + size_t queueSize = m_RenderQueue.size(); + ImGuiPlots::instance().observeFloat(PLOT_QUEUED_FRAMES, (float)queueSize); + } + + // notify render loop of new frame + m_RenderQueueNotEmpty.notify_one(); } void Pacer::renderFrame(std::shared_ptr& sceneRenderer, AVFrame* frame) @@ -82,7 +258,7 @@ void Pacer::renderFrame(std::shared_ptr& sceneRenderer, AVFrame* QueryPerformanceCounter(&afterRender); if (frame->opaque_ref) { - MLFrameData *data = (MLFrameData *)frame->opaque_ref->data; + auto* data = reinterpret_cast(frame->opaque_ref->data); m_DeviceResources->GetStats()->SubmitPacerTime( beforeRender.QuadPart - data->decodeEndQpc, @@ -92,9 +268,8 @@ void Pacer::renderFrame(std::shared_ptr& sceneRenderer, AVFrame* av_frame_free(&frame); // Drop frames if we have too many queued up for a while - m_FrameQueueLock.lock(); + std::unique_lock lock(m_FrameQueueLock); - // these were originally 0 and 2 int frameDropTarget = m_FrameDropTarget; for (int queueHistoryEntry : m_RenderQueueHistory) { if (queueHistoryEntry == 0) { @@ -119,42 +294,40 @@ void Pacer::renderFrame(std::shared_ptr& sceneRenderer, AVFrame* AVFrame* frame = std::move(m_RenderQueue.front()); m_RenderQueue.pop_front(); - FQLog("!! pacer queue overflow: %d > frame drop target %d, dropping oldest frame (pts %.3f ms)\n", - m_RenderQueue.size() + 1, frameDropTarget, (frame->pts / 90000.0) * 1000); + FQLog("! render queue overflow: %d > frame drop target %d, dropping oldest frame (pts %.3f ms)\n", + m_RenderQueue.size() + 1, frameDropTarget, frame->pts / 90.0); m_DeviceResources->GetStats()->SubmitDroppedFrame(1); // Drop the lock while we call av_frame_free() - m_FrameQueueLock.unlock(); + lock.unlock(); av_frame_free(&frame); - m_FrameQueueLock.lock(); + lock.lock(); } - m_FrameQueueLock.unlock(); + lock.unlock(); // Don't graph 0 here, because it's already being done in dropFrameForEnqueue if (dropCount > 0) { - ImGuiPlots::instance().observeFloat(PLOT_DROPPED_PACER, (float)dropCount); + ImGuiPlots::instance().observeFloat(PLOT_DROPPED_PACER_FRONT, (float)dropCount); } } -void Pacer::dropFrameForEnqueue(std::deque& queue) +void Pacer::dropFrameForEnqueue(std::deque& queue, int plotId) { - assert(queue.size() <= MAX_QUEUED_FRAMES); - int dropCount = 0; - if (queue.size() == MAX_QUEUED_FRAMES) { + if (queue.size() >= MAX_QUEUED_FRAMES) { AVFrame* frame = std::move(queue.front()); queue.pop_front(); - FQLog("!! pacer queue full (%d), dropping oldest frame (pts %.3f ms)\n", - queue.size() + 1, (frame->pts / 90000.0) * 1000); + FQLog("! pacing queue %d full (%d), dropping oldest frame (pts %.3f ms)\n", + plotId, queue.size() + 1, frame->pts / 90.0); m_DeviceResources->GetStats()->SubmitDroppedFrame(1); av_frame_free(&frame); dropCount = 1; } - ImGuiPlots::instance().observeFloat(PLOT_DROPPED_PACER, (float)dropCount); + ImGuiPlots::instance().observeFloat(plotId, (float)dropCount); } void Pacer::submitFrame(AVFrame* frame) @@ -165,15 +338,16 @@ void Pacer::submitFrame(AVFrame* frame) // Queue the frame and possibly wake up the render thread { std::scoped_lock lock(m_FrameQueueLock); - dropFrameForEnqueue(m_RenderQueue); - m_RenderQueue.push_back(frame); + + dropFrameForEnqueue(m_PacingQueue, PLOT_DROPPED_PACER_BACK); + m_PacingQueue.push_back(frame); size_t queueSize = m_RenderQueue.size(); ImGuiPlots::instance().observeFloat(PLOT_QUEUED_FRAMES, (float)queueSize); } - // notify render loop of new frame - m_RenderQueueNotEmpty.notify_one(); + // notify pacing loop of new frame + m_PacingQueueNotEmpty.notify_one(); } int Pacer::modifyFrameDropTarget(bool increase) { diff --git a/Streaming/Pacer.h b/Streaming/Pacer.h index 343a4b66..959928d2 100644 --- a/Streaming/Pacer.h +++ b/Streaming/Pacer.h @@ -1,16 +1,16 @@ #pragma once #include "VideoRenderer.h" +#include "Utils.hpp" #include #include +#include extern "C" { #include } -using namespace moonlight_xbox_dx; - class Pacer { public: @@ -24,20 +24,32 @@ class Pacer void waitForFrame(); - bool renderOnMainThread(std::shared_ptr& sceneRenderer); + bool renderOnMainThread(std::shared_ptr& sceneRenderer); int modifyFrameDropTarget(bool increase); private: - void renderFrame(std::shared_ptr& sceneRenderer, AVFrame* frame); + void vsyncThread(); + + void waitForVsync(); + + void handleVsync(double timeUntilNextVsyncMillis); + + void enqueueFrameForRenderingAndUnlock(AVFrame* frame); - void dropFrameForEnqueue(std::deque& queue); + void renderFrame(std::shared_ptr& sceneRenderer, AVFrame* frame); + + void dropFrameForEnqueue(std::deque& queue, int plotId); std::shared_ptr m_DeviceResources; + std::thread m_VsyncThread; std::deque m_RenderQueue; + std::deque m_PacingQueue; std::deque m_RenderQueueHistory; - std::condition_variable m_RenderQueueNotEmpty; + std::deque m_PacingQueueHistory; std::mutex m_FrameQueueLock; + std::condition_variable m_RenderQueueNotEmpty; + std::condition_variable m_PacingQueueNotEmpty; std::atomic m_Stopping{false}; int m_MaxVideoFps; double m_DisplayFps; diff --git a/Streaming/StatsRenderer.cpp b/Streaming/StatsRenderer.cpp index 2e1c5977..c0675014 100644 --- a/Streaming/StatsRenderer.cpp +++ b/Streaming/StatsRenderer.cpp @@ -73,14 +73,15 @@ void StatsRenderer::RenderGraphs() QueryPerformanceCounter(&t0); // we malloc a buffer for each stat only once and reuse it each frame - assert(PlotCount == 7); - static float *buffers[7] = { + assert(PlotCount == 8); + static float *buffers[8] = { (float *)malloc(sizeof(float) * 512), (float *)malloc(sizeof(float) * 512), (float *)malloc(sizeof(float) * 512), (float *)malloc(sizeof(float) * 512), (float *)malloc(sizeof(float) * 512), (float *)malloc(sizeof(float) * 512), + (float *)malloc(sizeof(float) * 512), (float *)malloc(sizeof(float) * 512)}; ImGuiIO &io = ImGui::GetIO(); @@ -167,15 +168,15 @@ void StatsRenderer::RenderGraphs() } ImGui::Dummy(ImVec2(1.0f, itemSpacingY)); - const int row2[3] = {PLOT_HOST_FRAMETIME, PLOT_DROPPED_PACER, PLOT_OVERHEAD}; + const int row2[3] = {PLOT_HOST_FRAMETIME, PLOT_DROPPED_PACER_BACK, PLOT_DROPPED_PACER_FRONT}; for (int c = 0; c < 3; ++c) { if (c > 0) ImGui::SameLine(0.0f, itemSpacingX); draw_plot(row2[c], graphW, graphH); } // 3rd row for quickly graphing something if needed - // ImGui::Dummy(ImVec2(1.0f, itemSpacingY)); - // draw_plot(PLOT_ETC, graphW, graphH); + ImGui::Dummy(ImVec2(1.0f, itemSpacingY)); + draw_plot(PLOT_OVERHEAD, graphW, graphH); ImGui::End(); diff --git a/Streaming/moonlight_xbox_dxMain.cpp b/Streaming/moonlight_xbox_dxMain.cpp index 8a8477b7..caca5f8f 100644 --- a/Streaming/moonlight_xbox_dxMain.cpp +++ b/Streaming/moonlight_xbox_dxMain.cpp @@ -116,18 +116,10 @@ void moonlight_xbox_dxMain::StartRenderLoop() FFMpegDecoder::instance().WaitForFrame(); Update(); - - // Make sure our use of D3D doesn't overlap with ffmpeg's hardware decoding - // LARGE_INTEGER t0, t1; - // FQLog("render loop before lock_context\n"); - // QueryPerformanceCounter(&t0); - // FFMpegDecoder::instance().mutex.lock(); - // QueryPerformanceCounter(&t1); - // FQLog("render loop acquired lock_context in %.3f ms\n", QpcToMs(t1.QuadPart - t0.QuadPart)); if (Render()) { QueryPerformanceCounter(&beforePresent); - static uint64_t lastPresentTime = 0; + static int64_t lastPresentTime = 0; if (lastPresentTime > 0) { double frametime = QpcToMs(beforePresent.QuadPart - lastPresentTime); ImGuiPlots::instance().observeFloat(PLOT_FRAMETIME, (float)frametime); @@ -136,8 +128,6 @@ void moonlight_xbox_dxMain::StartRenderLoop() m_deviceResources->Present(); } - //FFMpegDecoder::instance().mutex.unlock(); - //FQLog("render loop unlock context\n"); } }); m_renderLoopWorker = ThreadPool::RunAsync(workItemHandler, WorkItemPriority::High, WorkItemOptions::TimeSliced); diff --git a/moonlight-xbox-dx.vcxproj b/moonlight-xbox-dx.vcxproj index d0a7a906..78562272 100644 --- a/moonlight-xbox-dx.vcxproj +++ b/moonlight-xbox-dx.vcxproj @@ -257,6 +257,7 @@ $(ProjectDir)/libgamestream/build/$(Configuration)/gamestream.lib;$(ProjectDir)/third_party/DirectXTK/Bin/Windows10_2022/x64/$(Configuration)/DirectXTK.lib;libexpat.lib;libcurl.lib;libcrypto.lib;libssl.lib;avcodec.lib;avutil.lib;swscale.lib;d2d1.lib;d3d11.lib;dxgi.lib;windowscodecs.lib;dwrite.lib;opus.lib;%(AdditionalDependencies) %(AdditionalLibraryDirectories);$(ProjectDir)vcpkg_installed\x64-uwp\lib;$(VCInstallDir)\lib\store\amd64;$(VCInstallDir)\lib\amd64;$(ProjectDir)third_party\moonlight-common-c\build\$(Configuration)\ + true pch.h diff --git a/moonlight-xbox-dx.vcxproj.filters b/moonlight-xbox-dx.vcxproj.filters index 7cd00399..8ecfd554 100644 --- a/moonlight-xbox-dx.vcxproj.filters +++ b/moonlight-xbox-dx.vcxproj.filters @@ -2744,6 +2744,230 @@ DLLs + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + + DLLs + + + DLLs + diff --git a/pch.h b/pch.h index 1b6cddb4..491569be 100644 --- a/pch.h +++ b/pch.h @@ -43,36 +43,33 @@ ) // Time helpers -static uint64_t QpcToUs(uint64_t qpc) { +static int64_t QpcFreq() { static LARGE_INTEGER qpcFreq = {0, 0}; if (qpcFreq.QuadPart == 0) { QueryPerformanceFrequency(&qpcFreq); } - - return (qpc / qpcFreq.QuadPart) * UINT64_C(1000000) + - (qpc % qpcFreq.QuadPart) * UINT64_C(1000000) / qpcFreq.QuadPart; + return qpcFreq.QuadPart; } -static uint64_t QpcToNs(uint64_t qpc) { - // Convert QPC units (1/perf_freq seconds) to nanoseconds. This will work - // without overflow because the QPC value is guaranteed not to roll-over - // within 100 years, so perf_freq must be less than 2.9*10^9. - static LARGE_INTEGER qpcFreq = {0, 0}; - if (qpcFreq.QuadPart == 0) { - QueryPerformanceFrequency(&qpcFreq); - } +static int64_t QpcToUs(int64_t qpc) { + int64_t qpcFreq = QpcFreq(); + return (qpc / qpcFreq) * UINT64_C(1000000) + + (qpc % qpcFreq) * UINT64_C(1000000) / qpcFreq; +} - return (qpc / qpcFreq.QuadPart) * UINT64_C(1000000000) + - (qpc % qpcFreq.QuadPart) * UINT64_C(1000000000) / qpcFreq.QuadPart; +static int64_t QpcToNs(int64_t qpc) { + int64_t qpcFreq = QpcFreq(); + return (qpc / qpcFreq) * UINT64_C(1000000000) + + (qpc % qpcFreq) * UINT64_C(1000000000) / qpcFreq; } -static uint64_t QpcNsNow() { - LARGE_INTEGER perf_count; - QueryPerformanceCounter(&perf_count); - return QpcToNs(perf_count.QuadPart); +static int64_t QpcNsNow() { + LARGE_INTEGER perf_count; + QueryPerformanceCounter(&perf_count); + return QpcToNs(perf_count.QuadPart); } -static double QpcToMs(uint64_t qpc) { +static double QpcToMs(int64_t qpc) { return (double)QpcToUs(qpc) / 1000.0f; } From eca04d7ab7ad1168c631d67398e125a6ca3acf6c Mon Sep 17 00:00:00 2001 From: Andy Grundman Date: Thu, 18 Sep 2025 19:12:04 -0400 Subject: [PATCH 09/22] Fix queued frames graph for m_PacingQueue --- Streaming/Pacer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Streaming/Pacer.cpp b/Streaming/Pacer.cpp index 1265bd0e..9b5b4985 100644 --- a/Streaming/Pacer.cpp +++ b/Streaming/Pacer.cpp @@ -342,7 +342,7 @@ void Pacer::submitFrame(AVFrame* frame) dropFrameForEnqueue(m_PacingQueue, PLOT_DROPPED_PACER_BACK); m_PacingQueue.push_back(frame); - size_t queueSize = m_RenderQueue.size(); + size_t queueSize = m_PacingQueue.size(); ImGuiPlots::instance().observeFloat(PLOT_QUEUED_FRAMES, (float)queueSize); } From a271b89ae6dde4e18e93d6165de7e05cca2d0b64 Mon Sep 17 00:00:00 2001 From: Andy Grundman Date: Fri, 19 Sep 2025 09:11:29 -0400 Subject: [PATCH 10/22] Since I can't find a proper wait-for-vsync method, this should operate in 'front-only' pacer mode --- Pages/StreamPage.xaml | 2 +- Pages/StreamPage.xaml.cpp | 9 ++-- Streaming/FFmpegDecoder.cpp | 9 +++- Streaming/FFmpegDecoder.h | 1 + Streaming/Pacer.cpp | 73 ++++++++++++++++------------- Streaming/Pacer.h | 5 +- Streaming/StatsRenderer.cpp | 4 +- Streaming/moonlight_xbox_dxMain.cpp | 2 +- 8 files changed, 62 insertions(+), 43 deletions(-) diff --git a/Pages/StreamPage.xaml b/Pages/StreamPage.xaml index 7e341ea3..3ef01576 100644 --- a/Pages/StreamPage.xaml +++ b/Pages/StreamPage.xaml @@ -46,7 +46,7 @@ - + diff --git a/Pages/StreamPage.xaml.cpp b/Pages/StreamPage.xaml.cpp index c41a1878..16600076 100644 --- a/Pages/StreamPage.xaml.cpp +++ b/Pages/StreamPage.xaml.cpp @@ -231,15 +231,13 @@ void StreamPage::toggleHDR_WinAltB_Click(Platform::Object^ sender, Windows::UI:: void StreamPage::increaseFrameDropTarget_Click(Platform::Object^ sender, Windows::UI::Xaml::RoutedEventArgs^ e) { - int target = 1; - target = FFMpegDecoder::instance().ModifyFrameDropTarget(true); // true increases the value + int target = FFMpegDecoder::instance().ModifyFrameDropTarget(true); // true increases the value this->frameDropTargetLabel->Text = Utils::StringPrintf("Frame queue size: %d", target); } void StreamPage::decreaseFrameDropTarget_Click(Platform::Object^ sender, Windows::UI::Xaml::RoutedEventArgs^ e) { - int target = 1; - target = FFMpegDecoder::instance().ModifyFrameDropTarget(false); // false decreases the value + int target = FFMpegDecoder::instance().ModifyFrameDropTarget(false); // false decreases the value this->frameDropTargetLabel->Text = Utils::StringPrintf("Frame queue size: %d", target); } @@ -247,6 +245,9 @@ void StreamPage::OnLoaded(Platform::Object^ sender, Windows::UI::Xaml::RoutedEve { auto navigation = Windows::UI::Core::SystemNavigationManager::GetForCurrentView(); m_back_cookie = navigation->BackRequested += ref new EventHandler(this, &StreamPage::OnBackRequested); + + int target = FFMpegDecoder::instance().GetFrameDropTarget(); + this->frameDropTargetLabel->Text = Utils::StringPrintf("Frame queue size: %d", target); } diff --git a/Streaming/FFmpegDecoder.cpp b/Streaming/FFmpegDecoder.cpp index d235b7e1..4112cce5 100644 --- a/Streaming/FFmpegDecoder.cpp +++ b/Streaming/FFmpegDecoder.cpp @@ -88,7 +88,7 @@ namespace moonlight_xbox_dx { this->resources = res; m_Pacer = std::make_unique(res); - m_Pacer->initialize(config->fps, res->GetRefreshRate()); + m_Pacer->initialize(config->fps, res->GetRefreshRate(), false /* useVsyncThread */); } int FFMpegDecoder::Init(int videoFormat, int width, int height, int redrawRate, void* context, int drFlags) { @@ -316,6 +316,13 @@ namespace moonlight_xbox_dx { return DR_OK; } + int FFMpegDecoder::GetFrameDropTarget() { + if (m_Pacer) { + return m_Pacer->getFrameDropTarget(); + } + return 0; + } + int FFMpegDecoder::ModifyFrameDropTarget(bool increase) { return m_Pacer->modifyFrameDropTarget(increase); } diff --git a/Streaming/FFmpegDecoder.h b/Streaming/FFmpegDecoder.h index 286e81a9..f4db08eb 100644 --- a/Streaming/FFmpegDecoder.h +++ b/Streaming/FFmpegDecoder.h @@ -32,6 +32,7 @@ class FFMpegDecoder { int Init(int videoFormat, int width, int height, int redrawRate, void *context, int drFlags); void Cleanup(); int SubmitDecodeUnit(PDECODE_UNIT decodeUnit); + int GetFrameDropTarget(); int ModifyFrameDropTarget(bool increase); void WaitForFrame(); bool RenderFrameOnMainThread(std::shared_ptr &sceneRenderer); diff --git a/Streaming/Pacer.cpp b/Streaming/Pacer.cpp index 9b5b4985..e0d58dc3 100644 --- a/Streaming/Pacer.cpp +++ b/Streaming/Pacer.cpp @@ -40,10 +40,12 @@ Pacer::~Pacer() m_Stopping = true; m_RenderQueueNotEmpty.notify_all(); - // Stop the V-sync thread - m_PacingQueueNotEmpty.notify_all(); - if (m_VsyncThread.joinable()) { - m_VsyncThread.join(); + if (m_UseVsyncThread) { + // Stop the V-sync thread + m_PacingQueueNotEmpty.notify_all(); + if (m_VsyncThread.joinable()) { + m_VsyncThread.join(); + } } // Drain and free any remaining frames in both queues @@ -67,21 +69,25 @@ Pacer::~Pacer() } } -void Pacer::initialize(int maxVideoFps, double refreshRate) +void Pacer::initialize(int maxVideoFps, double refreshRate, bool useVsyncThread) { m_MaxVideoFps = maxVideoFps; m_DisplayFps = refreshRate; + m_UseVsyncThread = useVsyncThread; Utils::Logf("Frame pacing: target %.2f Hz with %d FPS stream", m_DisplayFps, m_MaxVideoFps); - // Start the dedicated vblank pacing thread once - if (!m_VsyncThread.joinable()) { - m_VsyncThread = std::thread(&Pacer::vsyncThread, this); + if (m_UseVsyncThread) { + // Start the dedicated vblank pacing thread + if (!m_VsyncThread.joinable()) { + m_VsyncThread = std::thread(&Pacer::vsyncThread, this); + } } } -// Vsync thread +// Vsync thread, this method needs a way to wait on vsync, and it's unclear how +// to do this in UWP. void Pacer::vsyncThread() { @@ -91,21 +97,13 @@ void Pacer::vsyncThread() while (!m_Stopping) { // See comments in waitForVsync() + waitForVsync(); - LARGE_INTEGER beforeWait, afterWait; - // QueryPerformanceCounter(&beforeWait); - // waitForVsync(); - // QueryPerformanceCounter(&afterWait); - // FQLog("waitForVsync() waited %.3f ms\n", QpcToMs(afterWait.QuadPart - beforeWait.QuadPart)); - - // if (m_Stopping) { - // break; - // } + if (m_Stopping) { + break; + } - QueryPerformanceCounter(&beforeWait); handleVsync(1000.0 / m_DisplayFps); - QueryPerformanceCounter(&afterWait); - FQLog("handleVsync() waited %.3f ms\n", QpcToMs(afterWait.QuadPart - beforeWait.QuadPart)); } } @@ -118,7 +116,7 @@ void Pacer::waitForVsync() //m_DeviceResources->GetDXGIOutput()->WaitForVBlank(); //HANDLE flw = m_DeviceResources->GetFrameLatencyWaitable(); - //WaitForSingleObject(flw, 2000 /*ms timeout*/); + //WaitForSingleObjectEx(flw, 1000, true); } // Called in an arbitrary thread by the IVsyncSource on V-sync @@ -319,8 +317,8 @@ void Pacer::dropFrameForEnqueue(std::deque& queue, int plotId) AVFrame* frame = std::move(queue.front()); queue.pop_front(); - FQLog("! pacing queue %d full (%d), dropping oldest frame (pts %.3f ms)\n", - plotId, queue.size() + 1, frame->pts / 90.0); + FQLog("! queue full (%d), dropping oldest frame (pts %.3f ms)\n", + queue.size() + 1, frame->pts / 90.0); m_DeviceResources->GetStats()->SubmitDroppedFrame(1); av_frame_free(&frame); @@ -335,19 +333,28 @@ void Pacer::submitFrame(AVFrame* frame) // Make sure initialize() has been called assert(m_MaxVideoFps != 0); - // Queue the frame and possibly wake up the render thread - { - std::scoped_lock lock(m_FrameQueueLock); + if (m_UseVsyncThread) { + // Queue the frame and possibly wake up the render thread + { + std::scoped_lock lock(m_FrameQueueLock); - dropFrameForEnqueue(m_PacingQueue, PLOT_DROPPED_PACER_BACK); - m_PacingQueue.push_back(frame); + dropFrameForEnqueue(m_PacingQueue, PLOT_DROPPED_PACER_BACK); + m_PacingQueue.push_back(frame); - size_t queueSize = m_PacingQueue.size(); - ImGuiPlots::instance().observeFloat(PLOT_QUEUED_FRAMES, (float)queueSize); + size_t queueSize = m_PacingQueue.size(); + ImGuiPlots::instance().observeFloat(PLOT_QUEUED_FRAMES, (float)queueSize); + } + + // notify pacing loop of new frame + m_PacingQueueNotEmpty.notify_one(); } + else { + enqueueFrameForRenderingAndUnlock(frame); + } +} - // notify pacing loop of new frame - m_PacingQueueNotEmpty.notify_one(); +int Pacer::getFrameDropTarget() { + return m_FrameDropTarget; } int Pacer::modifyFrameDropTarget(bool increase) { diff --git a/Streaming/Pacer.h b/Streaming/Pacer.h index 959928d2..2a2dde3f 100644 --- a/Streaming/Pacer.h +++ b/Streaming/Pacer.h @@ -20,12 +20,14 @@ class Pacer void submitFrame(AVFrame* frame); - void initialize(int maxVideoFps, double refreshRate); + void initialize(int maxVideoFps, double refreshRate, bool useVsyncThread); void waitForFrame(); bool renderOnMainThread(std::shared_ptr& sceneRenderer); + int getFrameDropTarget(); + int modifyFrameDropTarget(bool increase); private: @@ -53,5 +55,6 @@ class Pacer std::atomic m_Stopping{false}; int m_MaxVideoFps; double m_DisplayFps; + bool m_UseVsyncThread; std::atomic_int m_FrameDropTarget; }; diff --git a/Streaming/StatsRenderer.cpp b/Streaming/StatsRenderer.cpp index c0675014..1d7d00f9 100644 --- a/Streaming/StatsRenderer.cpp +++ b/Streaming/StatsRenderer.cpp @@ -168,7 +168,7 @@ void StatsRenderer::RenderGraphs() } ImGui::Dummy(ImVec2(1.0f, itemSpacingY)); - const int row2[3] = {PLOT_HOST_FRAMETIME, PLOT_DROPPED_PACER_BACK, PLOT_DROPPED_PACER_FRONT}; + const int row2[3] = {PLOT_HOST_FRAMETIME, PLOT_DROPPED_PACER_FRONT, PLOT_OVERHEAD}; for (int c = 0; c < 3; ++c) { if (c > 0) ImGui::SameLine(0.0f, itemSpacingX); draw_plot(row2[c], graphW, graphH); @@ -176,7 +176,7 @@ void StatsRenderer::RenderGraphs() // 3rd row for quickly graphing something if needed ImGui::Dummy(ImVec2(1.0f, itemSpacingY)); - draw_plot(PLOT_OVERHEAD, graphW, graphH); + draw_plot(PLOT_DROPPED_PACER_BACK, graphW, graphH); ImGui::End(); diff --git a/Streaming/moonlight_xbox_dxMain.cpp b/Streaming/moonlight_xbox_dxMain.cpp index caca5f8f..42772617 100644 --- a/Streaming/moonlight_xbox_dxMain.cpp +++ b/Streaming/moonlight_xbox_dxMain.cpp @@ -114,7 +114,6 @@ void moonlight_xbox_dxMain::StartRenderLoop() critical_section::scoped_lock lock(m_criticalSection); LARGE_INTEGER beforeWait, afterWait, beforePresent; - FFMpegDecoder::instance().WaitForFrame(); Update(); if (Render()) { QueryPerformanceCounter(&beforePresent); @@ -396,6 +395,7 @@ bool moonlight_xbox_dxMain::Render() // Render the scene objects. //LOCK_D3D("Render"); + FFMpegDecoder::instance().WaitForFrame(); bool shouldPresent = FFMpegDecoder::instance().RenderFrameOnMainThread(m_sceneRenderer); m_LogRenderer->Render(); m_statsTextRenderer->Render(showImGui); From d19d7ec911089762b57a4184381a9b4a650766ff Mon Sep 17 00:00:00 2001 From: Andy Grundman Date: Tue, 23 Sep 2025 17:37:56 -0400 Subject: [PATCH 11/22] Add emulated vsync thread to provide a stable signal matching the refresh rate. Rework pacing code around this system. --- Common/DeviceResources.cpp | 43 +- Common/DeviceResources.h | 2 - Common/DirectXHelper.cpp | 104 ++++ Common/DirectXHelper.h | 40 ++ Plot/ImGuiPlots.cpp | 2 +- Plot/PlotDesc.h | 4 +- Streaming/FFmpegDecoder.cpp | 4 +- Streaming/Pacer.cpp | 739 +++++++++++++++++----------- Streaming/Pacer.h | 78 +-- Streaming/PacerRational.h | 59 +++ Streaming/StatsRenderer.cpp | 16 +- Streaming/moonlight_xbox_dxMain.cpp | 12 +- Streaming/moonlight_xbox_dxMain.h | 4 + moonlight-xbox-dx.vcxproj | 2 + moonlight-xbox-dx.vcxproj.filters | 230 +++++++++ pch.h | 43 +- 16 files changed, 974 insertions(+), 408 deletions(-) create mode 100644 Common/DirectXHelper.cpp create mode 100644 Streaming/PacerRational.h diff --git a/Common/DeviceResources.cpp b/Common/DeviceResources.cpp index 4fec2756..03d0e52a 100644 --- a/Common/DeviceResources.cpp +++ b/Common/DeviceResources.cpp @@ -62,8 +62,8 @@ DX::DeviceResources::DeviceResources() : m_d3dFeatureLevel(D3D_FEATURE_LEVEL_11_1), m_d3dRenderTargetSize(), m_dxgiFactoryFlags(0), - m_enableVsync(true), - m_swapchainVsync(true), + m_enableVsync(false), + m_swapchainVsync(false), m_outputSize(), m_logicalSize(), m_nativeOrientation(DisplayOrientations::None), @@ -187,6 +187,8 @@ void DX::DeviceResources::CreateDeviceResources() if (FAILED(hr)) { + DX::ThrowIfFailed(hr); + // If the initialization fails, fall back to the WARP device. // For more information on WARP, see: // https://go.microsoft.com/fwlink/?LinkId=286690 @@ -223,7 +225,6 @@ void DX::DeviceResources::CreateWindowSizeDependentResources() ID3D11RenderTargetView* nullViews[] = {nullptr}; m_d3dContext->OMSetRenderTargets(ARRAYSIZE(nullViews), nullViews, nullptr); m_d3dRenderTargetView = nullptr; - m_d3dDepthStencilView = nullptr; m_d3dContext->Flush1(D3D11_CONTEXT_TYPE_ALL, nullptr); UpdateRenderTargetSize(); @@ -280,7 +281,7 @@ void DX::DeviceResources::CreateWindowSizeDependentResources() swapChainDesc.SampleDesc.Quality = 0; swapChainDesc.BufferUsage = DXGI_USAGE_RENDER_TARGET_OUTPUT; //Check moonlight-stream/moonlight-qt/app/streaming/video/ffmpeg-renderers/d3d11va.cpp for rationale - swapChainDesc.BufferCount = 5; + swapChainDesc.BufferCount = 3; swapChainDesc.SwapEffect = DXGI_SWAP_EFFECT_FLIP_DISCARD; swapChainDesc.Flags = DXGI_SWAP_CHAIN_FLAG_FRAME_LATENCY_WAITABLE_OBJECT; if (!m_enableVsync) swapChainDesc.Flags |= DXGI_SWAP_CHAIN_FLAG_ALLOW_TEARING; @@ -323,7 +324,7 @@ void DX::DeviceResources::CreateWindowSizeDependentResources() swapChain.As(&m_swapChain) ); - //m_swapChain->SetMaximumFrameLatency(1); + m_swapChain->SetMaximumFrameLatency(2); m_frameLatencyWaitable = m_swapChain->GetFrameLatencyWaitableObject(); // Associate swap chain with SwapChainPanel @@ -415,34 +416,6 @@ void DX::DeviceResources::CreateWindowSizeDependentResources() ) ); - // Create a depth stencil view for use with 3D rendering if needed. - CD3D11_TEXTURE2D_DESC1 depthStencilDesc( - DXGI_FORMAT_D24_UNORM_S8_UINT, - lround(m_d3dRenderTargetSize.Width), - lround(m_d3dRenderTargetSize.Height), - 1, // This depth stencil view has only one texture. - 1, // Use a single mipmap level. - D3D11_BIND_DEPTH_STENCIL - ); - - ComPtr depthStencil; - DX::ThrowIfFailed( - m_d3dDevice->CreateTexture2D1( - &depthStencilDesc, - nullptr, - &depthStencil - ) - ); - - CD3D11_DEPTH_STENCIL_VIEW_DESC depthStencilViewDesc(D3D11_DSV_DIMENSION_TEXTURE2D); - DX::ThrowIfFailed( - m_d3dDevice->CreateDepthStencilView( - depthStencil.Get(), - &depthStencilViewDesc, - &m_d3dDepthStencilView - ) - ); - // Set the 3D rendering viewport to target the entire window. m_screenViewport = CD3D11_VIEWPORT( 0.0f, @@ -655,9 +628,6 @@ void DX::DeviceResources::Present() // overwritten. If dirty or scroll rects are used, this call should be modified. m_d3dContext->DiscardView1(m_d3dRenderTargetView.Get(), nullptr, 0); - // Discard the contents of the depth stencil. - m_d3dContext->DiscardView1(m_d3dDepthStencilView.Get(), nullptr, 0); - //UNLOCK_D3D(); // If the device was removed either by a disconnection or a driver upgrade, we @@ -810,6 +780,7 @@ double DX::DeviceResources::GetUWPRefreshRate() } else { // It seems difficult to get the refresh rate in Windows, TODO + refreshRate = 120.0; } return refreshRate; diff --git a/Common/DeviceResources.h b/Common/DeviceResources.h index ba541def..6181e593 100644 --- a/Common/DeviceResources.h +++ b/Common/DeviceResources.h @@ -65,7 +65,6 @@ namespace DX auto GetDXGIFactory() const noexcept { return m_dxgiFactory.Get(); } D3D_FEATURE_LEVEL GetDeviceFeatureLevel() const { return m_d3dFeatureLevel; } ID3D11RenderTargetView1* GetBackBufferRenderTargetView() const { return m_d3dRenderTargetView.Get(); } - ID3D11DepthStencilView* GetDepthStencilView() const { return m_d3dDepthStencilView.Get(); } DXGI_FORMAT GetBackBufferFormat() const noexcept { return m_backBufferFormat; } D3D11_VIEWPORT GetScreenViewport() const { return m_screenViewport; } DirectX::XMFLOAT4X4 GetOrientationTransform3D() const { return m_orientationTransform3D; } @@ -92,7 +91,6 @@ namespace DX // Direct3D rendering objects. Required for 3D. Microsoft::WRL::ComPtr m_d3dRenderTargetView; - Microsoft::WRL::ComPtr m_d3dDepthStencilView; D3D11_VIEWPORT m_screenViewport; // Cached reference to the XAML panel. diff --git a/Common/DirectXHelper.cpp b/Common/DirectXHelper.cpp new file mode 100644 index 00000000..12e11fca --- /dev/null +++ b/Common/DirectXHelper.cpp @@ -0,0 +1,104 @@ +#include "pch.h" +#include "Common\DirectXHelper.h" +#include "Common\DeviceResources.h" +#include + +using namespace Microsoft::WRL; + +DX::GpuPerformanceTimer::GpuPerformanceTimer( + const std::shared_ptr& deviceResources) : + m_deviceResources(deviceResources) +{ + ID3D11Device* device = m_deviceResources->GetD3DDevice(); + + const D3D11_QUERY_DESC timestampQuery = CD3D11_QUERY_DESC(D3D11_QUERY_TIMESTAMP); + const D3D11_QUERY_DESC disjointQuery = CD3D11_QUERY_DESC(D3D11_QUERY_TIMESTAMP_DISJOINT); + + for (uint32_t i = 0; i < QueryCount; ++i) + { + device->CreateQuery(×tampQuery, m_startTimestampQuery[i].ReleaseAndGetAddressOf()); + device->CreateQuery(×tampQuery, m_endTimestampQuery[i].ReleaseAndGetAddressOf()); + device->CreateQuery(&disjointQuery, m_disjointQuery[i].ReleaseAndGetAddressOf()); + } +} + +void DX::GpuPerformanceTimer::StartTimerForFrame() +{ + ID3D11DeviceContext* context = m_deviceResources->GetD3DDeviceContext(); + + // Just some book-keeping to see how long this algorithm is running + m_currentQuery = (m_currentQuery + 1) % QueryCount; + const uint32_t ReadQuery = (m_currentQuery + 1) % QueryCount; + + if (m_currentFrameIndex >= QueryCount) + { + D3D11_QUERY_DATA_TIMESTAMP_DISJOINT disjointData; + while (context->GetData(m_disjointQuery[ReadQuery].Get(), &disjointData, sizeof(disjointData), 0) != S_OK); + + if (disjointData.Disjoint == 0) + { + uint64_t start; + while (context->GetData(m_startTimestampQuery[ReadQuery].Get(), &start, sizeof(start), 0) != S_OK); + + uint64_t end; + while (context->GetData(m_endTimestampQuery[ReadQuery].Get(), &end, sizeof(end), 0) != S_OK); + + constexpr float secondsToMilliseconds = 1000.0f; + const float ticksToSeconds = 1.0f / static_cast(disjointData.Frequency); + const uint64_t ticksElapsed = end - start; + + m_processingTimeMs = static_cast(ticksElapsed) * ticksToSeconds * secondsToMilliseconds; + + // Reset every ~10 seconds so the min/max has a chance to update + if (m_currentFrameIndex % 300 == 0) + { + m_processingTimeMinMs = +std::numeric_limits::infinity(); + m_processingTimeMaxMs = -std::numeric_limits::infinity(); + } + + m_processingTimeMinMs = std::min(m_processingTimeMinMs, m_processingTimeMs); + m_processingTimeMaxMs = std::max(m_processingTimeMaxMs, m_processingTimeMs); + } + else + { + m_processingTimeMs = std::numeric_limits::quiet_NaN();; + } + + m_processingTimeHistory[m_processingTimeHistoryIndex] = m_processingTimeMs; + m_processingTimeHistoryIndex = (m_processingTimeHistoryIndex + 1) % TimeHistoryCount; + + m_processingTimeAvgMs = std::accumulate(m_processingTimeHistory, m_processingTimeHistory + TimeHistoryCount, 0.0f) / static_cast(TimeHistoryCount); + } + + ++m_currentFrameIndex; + + context->Begin(m_disjointQuery[m_currentQuery].Get()); + context->End(m_startTimestampQuery[m_currentQuery].Get()); +} + +void DX::GpuPerformanceTimer::EndTimerForFrame() +{ + ID3D11DeviceContext* context = m_deviceResources->GetD3DDeviceContext(); + context->End(m_endTimestampQuery[m_currentQuery].Get()); + context->End(m_disjointQuery[m_currentQuery].Get()); +} + +float DX::GpuPerformanceTimer::GetFrameTime() const +{ + return m_processingTimeMs; +} + +float DX::GpuPerformanceTimer::GetAvgFrameTime() const +{ + return m_processingTimeAvgMs; +} + +float DX::GpuPerformanceTimer::GetMinFrameTime() const +{ + return m_processingTimeMinMs; +} + +float DX::GpuPerformanceTimer::GetMaxFrameTime() const +{ + return m_processingTimeMaxMs; +} diff --git a/Common/DirectXHelper.h b/Common/DirectXHelper.h index 69fd228e..93c0460e 100644 --- a/Common/DirectXHelper.h +++ b/Common/DirectXHelper.h @@ -78,4 +78,44 @@ namespace DX return SUCCEEDED(hr); } #endif + + class DeviceResources; + + // Used to track time spent executing Gpu work + class GpuPerformanceTimer + { + public: + GpuPerformanceTimer( + const std::shared_ptr& deviceResources); + + void StartTimerForFrame(); + void EndTimerForFrame(); + + float GetFrameTime() const; + float GetAvgFrameTime() const; + float GetMinFrameTime() const; + float GetMaxFrameTime() const; + + private: + std::shared_ptr m_deviceResources; + + static constexpr uint32_t QueryCount = 5u; + + Microsoft::WRL::ComPtr m_startTimestampQuery[QueryCount]; + Microsoft::WRL::ComPtr m_endTimestampQuery[QueryCount]; + Microsoft::WRL::ComPtr m_disjointQuery[QueryCount]; + + uint32_t m_currentQuery = 0u; + uint32_t m_currentFrameIndex = 0u; + + static constexpr uint32_t TimeHistoryCount = 64u; + + uint32_t m_processingTimeHistoryIndex = 0u; + float m_processingTimeHistory[TimeHistoryCount] = {}; + + float m_processingTimeMs = 0.0f; + float m_processingTimeAvgMs = 0.0f; + float m_processingTimeMinMs = +std::numeric_limits::infinity(); + float m_processingTimeMaxMs = -std::numeric_limits::infinity(); + }; } diff --git a/Plot/ImGuiPlots.cpp b/Plot/ImGuiPlots.cpp index 96a99745..b002b9df 100644 --- a/Plot/ImGuiPlots.cpp +++ b/Plot/ImGuiPlots.cpp @@ -21,7 +21,7 @@ ImGuiPlots::ImGuiPlots() : plots_{{ Plot(kPlotDescs[PLOT_FRAMETIME]), Plot(kPlotDescs[PLOT_HOST_FRAMETIME]), - Plot(kPlotDescs[PLOT_QUEUED_FRAMES]), + Plot(kPlotDescs[PLOT_VSYNC_INTERVAL]), Plot(kPlotDescs[PLOT_DROPPED_NETWORK]), Plot(kPlotDescs[PLOT_DROPPED_PACER_BACK]), Plot(kPlotDescs[PLOT_DROPPED_PACER_FRONT]), diff --git a/Plot/PlotDesc.h b/Plot/PlotDesc.h index 240ac4f7..62a60eec 100644 --- a/Plot/PlotDesc.h +++ b/Plot/PlotDesc.h @@ -7,7 +7,7 @@ enum PlotType { PLOT_FRAMETIME = 0, PLOT_HOST_FRAMETIME, - PLOT_QUEUED_FRAMES, + PLOT_VSYNC_INTERVAL, PLOT_DROPPED_NETWORK, PLOT_DROPPED_PACER_BACK, PLOT_DROPPED_PACER_FRONT, @@ -38,7 +38,7 @@ struct PlotDesc inline constexpr std::array kPlotDescs = {{ {"Frametime", PLOT_LABEL_MIN_MAX_AVG, "ms", -0.1f, 50.0f, 0.0f, 49.9f}, {"Host Frametime", PLOT_LABEL_MIN_MAX_AVG, "ms", -0.1f, 50.0f, 0.0f, 49.9f}, - {"Queued Frames", PLOT_LABEL_MIN_MAX_AVG, "", -1.0f, 10.0f, 0.0f, 0.0f}, + {"Vsync interval", PLOT_LABEL_MIN_MAX_AVG, "ms", -0.1f, 20.0f, 0.0f, 0.0f}, {"Dropped Frames (network)", PLOT_LABEL_TOTAL_INT, "", -1.0f, 3.0f, 0.0f, 0.0f}, {"Dropped Frames (back pacing)", PLOT_LABEL_TOTAL_INT, "", -1.0f, 3.0f, 0.0f, 0.0f}, {"Dropped Frames (front pacing)", PLOT_LABEL_TOTAL_INT, "", -1.0f, 3.0f, 0.0f, 0.0f}, diff --git a/Streaming/FFmpegDecoder.cpp b/Streaming/FFmpegDecoder.cpp index 4112cce5..e75bfa6c 100644 --- a/Streaming/FFmpegDecoder.cpp +++ b/Streaming/FFmpegDecoder.cpp @@ -88,7 +88,7 @@ namespace moonlight_xbox_dx { this->resources = res; m_Pacer = std::make_unique(res); - m_Pacer->initialize(config->fps, res->GetRefreshRate(), false /* useVsyncThread */); + m_Pacer->initialize(config->fps, res->GetRefreshRate()); } int FFMpegDecoder::Init(int videoFormat, int width, int height, int redrawRate, void* context, int drFlags) { @@ -297,7 +297,7 @@ namespace moonlight_xbox_dx { FQLog("✓ Frame decoded [pts: %.3f] [in#: %d] [out#: %d] [lost: %d] decode time %.3f ms\n", frame->pts / 90.0, decodeUnit->frameNumber, decoder_ctx->frame_num, - decoder_ctx->frame_num - decodeUnit->frameNumber, + decodeUnit->frameNumber - decoder_ctx->frame_num, QpcToMs(decodeEnd.QuadPart - decodeStart.QuadPart)); // Even though we have a valid frame, the ffmpeg API needs us to loop and call avcodec_receive_frame() diff --git a/Streaming/Pacer.cpp b/Streaming/Pacer.cpp index e0d58dc3..a3144b07 100644 --- a/Streaming/Pacer.cpp +++ b/Streaming/Pacer.cpp @@ -1,11 +1,42 @@ -#include "pch.h" #include "Pacer.h" -#include "FFmpegDecoder.h" -#include "../Plot/ImGuiPlots.h" -#include "Utils.hpp" -#include +#include "pch.h" #include #include +#include +#include "../Plot/ImGuiPlots.h" +#include "FFmpegDecoder.h" +#include "PacerRational.h" +#include "Utils.hpp" + +// Frame Pacing operation +// 4 threads use this class: +// +// Decoder thread (run from moonlight-common-c because DIRECT_SUBMIT) +// * submits AVFrame frames to pacing queue via submitFrame() +// * max frames in pacing queue = 4, will drop oldest frame if it's full (dropped by back pacing) +// +// vsyncEmulator thread: +// * Creates our own 120hz vsync signal since Xbox UWP only supports 60hz +// * attempts to send m_VsyncSignalled as accurately locked to the refresh rate as possible. +// * uses rational integer math to support 59.94 and 119.88 +// * syncs periodically with hardware 60hz vsync to prevent drift +// +// backPacer thread (called vsyncThread in moonlight-qt) +// * wakes up on each signal from vsync thread, uses same deadline Qpc as emulator thread +// * maintains pacing queue +// * moves 1 frame to render queue +// * tries to accurately time signal to main thread +// * signals main thread to start rendering +// +// renderOnMainThread +// * called from main render loop after waiting on GetFrameLatencyWaitableObject and signal from backPacer +// * calls frontPacer() (renderFrame in moonlight-qt) +// * maintains render queue +// +// Other differences from moonlight-qt: +// SwapChain BufferCount = 3 +// GetFrameLatencyWaitableObject() is used to wait before each frame +// SetMaximumFrameLatency(1) (TODO: with dynamic adjustment to 2 if necessary?) // Limit the number of queued frames to prevent excessive memory consumption // if the V-Sync source or renderer is blocked for a while. It's important @@ -19,356 +50,468 @@ // the next V-sync period. It also takes some amount of time // to do the render itself, so we can't render right before // V-sync happens. -#define TIMER_SLACK_MS 3 +#define TIMER_SLACK_MS 2 using steady_clock = std::chrono::steady_clock; using namespace moonlight_xbox_dx; -Pacer::Pacer(const std::shared_ptr& res) : +Pacer::Pacer(const std::shared_ptr &res) : m_DeviceResources(res), - m_VsyncThread(), + m_BackPacerThread(), + m_VsyncSeq(0), + m_VsyncNextDeadlineQpc(0), m_Stopping(false), m_MaxVideoFps(0), - m_DisplayFps(0.0), - m_FrameDropTarget(0) -{ + m_RefreshRate(0.0), + m_FrameDropTarget(0) { +} +Pacer::~Pacer() { + m_Stopping.store(true, std::memory_order_release); + m_RenderQueueNotEmpty.notify_all(); + + // Stop the vsync threads + m_VsyncSignalled.notify_all(); + m_PacingQueueNotEmpty.notify_all(); + if (m_BackPacerThread.joinable()) { + m_BackPacerThread.join(); + } + if (m_VsyncEmulator.joinable()) { + m_VsyncEmulator.join(); + } + + // Drain and free any remaining frames in both queues + { + std::unique_lock lock(m_FrameQueueLock); + + while (!m_RenderQueue.empty()) { + AVFrame *f = m_RenderQueue.front(); + m_RenderQueue.pop_front(); + lock.unlock(); + av_frame_free(&f); + lock.lock(); + } + while (!m_PacingQueue.empty()) { + AVFrame *f = m_PacingQueue.front(); + m_PacingQueue.pop_front(); + lock.unlock(); + av_frame_free(&f); + lock.lock(); + } + } } -Pacer::~Pacer() -{ - m_Stopping = true; - m_RenderQueueNotEmpty.notify_all(); - - if (m_UseVsyncThread) { - // Stop the V-sync thread - m_PacingQueueNotEmpty.notify_all(); - if (m_VsyncThread.joinable()) { - m_VsyncThread.join(); - } - } - - // Drain and free any remaining frames in both queues - { - std::unique_lock lock(m_FrameQueueLock); - - while (!m_RenderQueue.empty()) { - AVFrame* f = m_RenderQueue.front(); - m_RenderQueue.pop_front(); - lock.unlock(); - av_frame_free(&f); - lock.lock(); - } - while (!m_PacingQueue.empty()) { - AVFrame* f = m_PacingQueue.front(); - m_PacingQueue.pop_front(); - lock.unlock(); - av_frame_free(&f); - lock.lock(); - } - } +void Pacer::initialize(int maxVideoFps, double refreshRate) { + m_MaxVideoFps = maxVideoFps; + m_RefreshRate = refreshRate; + + Utils::Logf("Frame pacing: target %.2f Hz with %d FPS stream", + m_RefreshRate, m_MaxVideoFps); + + // Start the timing emulator thread + if (!m_VsyncEmulator.joinable()) { + m_VsyncEmulator = std::thread(&Pacer::vsyncEmulator, this); + } + + // Start the dedicated vblank pacing thread + if (!m_BackPacerThread.joinable()) { + m_BackPacerThread = std::thread(&Pacer::backPacer, this); + } +} + +// Vsync Emulator + +// Sleep until approximately targetQpc, then busy-wait to land precisely. +// sleepSlackUs: how early (in microseconds) to stop sleeping and start spinning +// tightSpinUs: within this final window, use a tight YieldProcessor loop +static inline void SleepUntilQpc(int64_t targetQpc, + int64_t sleepSlackUs = 1500, + int64_t tightSpinUs = 150) { + const int64_t f = QpcFreq(); + const int64_t sleepSlackQpc = UsToQpc(sleepSlackUs); + const int64_t tightSpinQpc = UsToQpc(tightSpinUs); + + for (;;) { + const int64_t now = QpcNow(); + int64_t remaining = targetQpc - now; + if (remaining <= 0) break; + + if (remaining > sleepSlackQpc) { + int64_t toSleepQpc = remaining - sleepSlackQpc; + DWORD ms = (DWORD)((toSleepQpc * 1000) / f); + if (ms == 0) ms = 1; + Sleep(ms); + continue; + } + + if (remaining > tightSpinQpc) { + YieldProcessor(); + continue; + } + + do { + YieldProcessor(); + } while ((targetQpc - QpcNow()) > 0); + break; + } } -void Pacer::initialize(int maxVideoFps, double refreshRate, bool useVsyncThread) -{ - m_MaxVideoFps = maxVideoFps; - m_DisplayFps = refreshRate; - m_UseVsyncThread = useVsyncThread; - - Utils::Logf("Frame pacing: target %.2f Hz with %d FPS stream", - m_DisplayFps, m_MaxVideoFps); - - if (m_UseVsyncThread) { - // Start the dedicated vblank pacing thread - if (!m_VsyncThread.joinable()) { - m_VsyncThread = std::thread(&Pacer::vsyncThread, this); - } - } +void Pacer::vsyncEmulator() { + if (!SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_HIGHEST)) { + Utils::Logf("Failed to set vsyncEmulator priority: %d\n", GetLastError()); + } + + QpcRationalPeriod period; + uint32_t hzNum = 60; + uint32_t hzDen = 1; + + switch ((int)std::round(m_RefreshRate * 100)) { + case 5994: + hzNum = 60000; + hzDen = 1001; + break; + case 6000: + hzNum = 60; + hzDen = 1; + break; + case 11988: + hzNum = 120000; + hzDen = 1001; + break; + case 12000: + hzNum = 120; + hzDen = 1; + break; + } + + Utils::Logf("Pacer: emulating vsync for %.2fhz using integer rate %lu/%lu\n", + m_RefreshRate, hzNum, hzDen); + + static constexpr int64_t remeasureEveryFrames = 3600; // 30s at 120hz + uint64_t frames = 0; + int64_t baseQpc = 0; + int64_t lastVsync = 0; + + while (!stopping()) { + int64_t curDeadlineQpc = period.nextDeadlineQpc; + + if (baseQpc == 0 || frames % remeasureEveryFrames == 0) { + // Align to a real vblank edge at startup and at regular intervals + m_DeviceResources->GetDXGIOutput()->WaitForVBlank(); + baseQpc = QpcNow(); + period.initFromHz(hzNum, hzDen, baseQpc); + + curDeadlineQpc = baseQpc; + } else { + // Emulate the vsync interval + SleepUntilQpc(period.nextDeadlineQpc); + } + + // Compute next period + period.step(); + ++frames; + + // signal vsync thread, passing it the next deadline + { + std::lock_guard lock(m_VsyncMutex); + ++m_VsyncSeq; + m_VsyncNextDeadlineQpc = period.nextDeadlineQpc; + } + m_VsyncSignalled.notify_one(); + + ImGuiPlots::instance().observeFloat(PLOT_VSYNC_INTERVAL, (float)QpcToMs(period.nextDeadlineQpc - curDeadlineQpc)); + } } -// Vsync thread, this method needs a way to wait on vsync, and it's unclear how -// to do this in UWP. +void Pacer::backPacer() { + if (!SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_ABOVE_NORMAL)) { + Utils::Logf("Failed to set vsyncThread priority: %d\n", GetLastError()); + } -void Pacer::vsyncThread() -{ - if (!SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_TIME_CRITICAL)) { - Utils::Logf("Failed to set vsyncThread priority: %d\n", GetLastError()); - } + uint64_t seenSeq = 0; - while (!m_Stopping) { - // See comments in waitForVsync() - waitForVsync(); + for (;;) { + int64_t nextDeadlineQpc = 0; - if (m_Stopping) { - break; - } + // Wait for either stop or a new vsync tick + { + std::unique_lock lock(m_VsyncMutex); + m_VsyncSignalled.wait(lock, [this, &seenSeq] { + return stopping() || (m_VsyncSeq != seenSeq); + }); - handleVsync(1000.0 / m_DisplayFps); - } -} + if (stopping()) break; -void Pacer::waitForVsync() -{ - // XXX for some reason, both of these vsync methods seem to run at 60hz even when Xbox is running at 120hz - // We can probably just run without them, but perfect smoothness probably requires some vsync support + seenSeq = m_VsyncSeq; + nextDeadlineQpc = m_VsyncNextDeadlineQpc; + } - // might work but is tied to a single display - //m_DeviceResources->GetDXGIOutput()->WaitForVBlank(); + handleVsync(nextDeadlineQpc); - //HANDLE flw = m_DeviceResources->GetFrameLatencyWaitable(); - //WaitForSingleObjectEx(flw, 1000, true); + LARGE_INTEGER t0; + static int64_t lastT0 = 0; + QueryPerformanceCounter(&t0); + FQLog("vsyncThread interval %.3f ms\n", QpcToMs(t0.QuadPart - lastT0)); + lastT0 = t0.QuadPart; + } } -// Called in an arbitrary thread by the IVsyncSource on V-sync -// or an event synchronized with V-sync -void Pacer::handleVsync(double timeUntilNextVsyncMillis) -{ - // Make sure initialize() has been called - assert(m_MaxVideoFps != 0); - - std::unique_lock lock(m_FrameQueueLock); - - // If the queue length history entries are large, be strict - // about dropping excess frames. - int frameDropTarget = 1; - - // If we may get more frames per second than we can display, use - // frame history to drop frames only if consistently above the - // one queued frame mark. - if (m_MaxVideoFps >= m_DisplayFps) { - for (int queueHistoryEntry : m_PacingQueueHistory) { - if (queueHistoryEntry <= 1) { - // Be lenient as long as the queue length - // resolves before the end of frame history - frameDropTarget = 3; - break; - } - } - - // Keep a rolling 500 ms window of pacing queue history - if (m_PacingQueueHistory.size() == m_DisplayFps / 2) { - m_PacingQueueHistory.pop_front(); - } - - m_PacingQueueHistory.push_back((int)m_PacingQueue.size()); - } - - // Catch up if we're several frames ahead - int dropCount = 0; - while (m_PacingQueue.size() > frameDropTarget) { - ++dropCount; - AVFrame* frame = std::move(m_PacingQueue.front()); - m_PacingQueue.pop_front(); - - FQLog("! pacing queue overflow: %d > frame drop target %d, dropping oldest frame (pts %.3f ms)\n", - m_PacingQueue.size() + 1, frameDropTarget, frame->pts / 90.0); +// XXX rename this +void Pacer::handleVsync(int64_t nextDeadlineQpc) { + // Compute remaining time until the next edge, then subtract slack. + const int64_t now = QpcNow(); + int64_t remainingQpc = nextDeadlineQpc - now; + if (remainingQpc < 0) remainingQpc = 0; + + const int64_t slackQpc = UsToQpc((int64_t)TIMER_SLACK_MS * 1000); + int64_t waitQpc = (remainingQpc > slackQpc) ? (remainingQpc - slackQpc) : 0; + const auto waitUs = std::chrono::microseconds(QpcToUs(waitQpc)); + + std::unique_lock lock(m_FrameQueueLock); + + // If the queue length history entries are large, be strict + // about dropping excess frames. + int frameDropTarget = 1; + + // If we may get more frames per second than we can display, use + // frame history to drop frames only if consistently above the + // one queued frame mark. + // XXX this should match for 119.88 now + if (m_MaxVideoFps >= m_RefreshRate) { + for (int queueHistoryEntry : m_PacingQueueHistory) { + if (queueHistoryEntry <= 1) { + // Be lenient as long as the queue length + // resolves before the end of frame history + frameDropTarget = 3; + break; + } + } + + // Keep a rolling 500 ms window of pacing queue history + if (m_PacingQueueHistory.size() == m_RefreshRate / 2) { + m_PacingQueueHistory.pop_front(); + } + + m_PacingQueueHistory.push_back((int)m_PacingQueue.size()); + } + + // Catch up if we're several frames ahead + int dropCount = 0; + while (m_PacingQueue.size() > frameDropTarget) { + ++dropCount; + AVFrame *frame = std::move(m_PacingQueue.front()); + m_PacingQueue.pop_front(); + + FQLog("! pacing queue overflow: %d > frame drop target %d, dropping oldest frame (pts %.3f ms)\n", + m_PacingQueue.size() + 1, frameDropTarget, frame->pts / 90.0); m_DeviceResources->GetStats()->SubmitDroppedFrame(1); - // Drop the lock while we call av_frame_free() - lock.unlock(); - av_frame_free(&frame); - lock.lock(); - } - - // Don't graph 0 here, because it's already being done in dropFrameForEnqueue - if (dropCount > 0) { - ImGuiPlots::instance().observeFloat(PLOT_DROPPED_PACER_BACK, (float)dropCount); - } - - if (m_PacingQueue.empty()) { - // Sleep up to the next vblank interval or until a frame arrives - static constexpr auto kSlack = std::chrono::milliseconds(TIMER_SLACK_MS); - const auto period_us = std::chrono::microseconds(static_cast(timeUntilNextVsyncMillis * 1000.0 + 0.5)); - const auto remaining = (period_us > kSlack) - ? (period_us - kSlack) - : std::chrono::microseconds(0); - - m_PacingQueueNotEmpty.wait_for(lock, remaining, [this]{ - return m_Stopping || !m_PacingQueue.empty(); - }); - - if (m_Stopping || m_PacingQueue.empty()) { - return; - } - } - - // Place the first frame on the render queue - AVFrame* frame = std::move(m_PacingQueue.front()); - m_PacingQueue.pop_front(); - lock.unlock(); - - enqueueFrameForRenderingAndUnlock(frame); + // Drop the lock while we call av_frame_free() + lock.unlock(); + av_frame_free(&frame); + lock.lock(); + } + + // Don't graph 0 here, because it's already being done in dropFrameForEnqueue + if (dropCount > 0) { + ImGuiPlots::instance().observeFloat(PLOT_DROPPED_PACER_BACK, (float)dropCount); + } + + if (m_PacingQueue.empty()) { + const int64_t t0 = QpcNow(); + m_PacingQueueNotEmpty.wait_for(lock, waitUs, [this] { + return stopping() || !m_PacingQueue.empty(); + }); + const int64_t t1 = QpcNow(); + FQLog("handleVsync waited for m_PacingQueueNotEmpty for %.3f ms\n", QpcToMs(t1 - t0)); + + if (stopping() || m_PacingQueue.empty()) { + return; + } + } else { + FQLog("handleVsync handling pacingQueue of size %d\n", m_PacingQueue.size()); + } + + // Place the first frame on the render queue + AVFrame *frame = std::move(m_PacingQueue.front()); + m_PacingQueue.pop_front(); + lock.unlock(); + + // try to time this as close to deadline as possible + remainingQpc = nextDeadlineQpc - QpcNow(); + if (remainingQpc > 150) { + SleepUntilQpc(nextDeadlineQpc); + } + + enqueueFrameForRenderingAndUnlock(frame); } // Main thread -void Pacer::waitForFrame() -{ - std::unique_lock lock(m_FrameQueueLock); - m_RenderQueueNotEmpty.wait(lock, [this]{ - return m_Stopping || !m_RenderQueue.empty(); - }); -} +void Pacer::waitForFrame() { + // Wait for the renderer to be ready for the next frame + LARGE_INTEGER t0, t1; + QueryPerformanceCounter(&t0); -bool Pacer::renderOnMainThread(std::shared_ptr& sceneRenderer) -{ - AVFrame* frame = nullptr; + HANDLE flw = m_DeviceResources->GetFrameLatencyWaitable(); + WaitForSingleObjectEx(flw, 1000, true); - { - std::scoped_lock lock(m_FrameQueueLock); - if (!m_RenderQueue.empty()) { - frame = std::move(m_RenderQueue.front()); - m_RenderQueue.pop_front(); - } - } + QueryPerformanceCounter(&t1); + FQLog("waitForFrame part 1 waited %.3f ms\n", QpcToMs(t1.QuadPart - t0.QuadPart)); - if (!frame) { - return false; // no frame, don't Present() - } + std::unique_lock lock(m_FrameQueueLock); + m_RenderQueueNotEmpty.wait(lock, [this] { + return stopping() || !m_RenderQueue.empty(); + }); - renderFrame(sceneRenderer, frame); - return true; // ok to Present() + int64_t t2 = QpcNow(); + FQLog("waitForFrame part 2 waited %.3f ms\n", QpcToMs(t2 - t1.QuadPart)); } -void Pacer::enqueueFrameForRenderingAndUnlock(AVFrame *frame) -{ - { - std::scoped_lock lock(m_FrameQueueLock); - dropFrameForEnqueue(m_RenderQueue, PLOT_DROPPED_PACER_FRONT); - m_RenderQueue.push_back(frame); +bool Pacer::renderOnMainThread(std::shared_ptr &sceneRenderer) { + AVFrame *frame = nullptr; + + { + std::scoped_lock lock(m_FrameQueueLock); + if (!m_RenderQueue.empty()) { + frame = std::move(m_RenderQueue.front()); + m_RenderQueue.pop_front(); + } + } + + if (!frame) { + return false; // no frame, don't Present() + } + + frontPacer(sceneRenderer, frame); + return true; // ok to Present() +} - size_t queueSize = m_RenderQueue.size(); - ImGuiPlots::instance().observeFloat(PLOT_QUEUED_FRAMES, (float)queueSize); - } +void Pacer::enqueueFrameForRenderingAndUnlock(AVFrame *frame) { + { + std::scoped_lock lock(m_FrameQueueLock); + dropFrameForEnqueue(m_RenderQueue, PLOT_DROPPED_PACER_FRONT); + m_RenderQueue.push_back(frame); + } - // notify render loop of new frame - m_RenderQueueNotEmpty.notify_one(); + // notify render loop of new frame + FQLog("enqueueFrame notifying m_RenderQueueNotEmpty\n"); + m_RenderQueueNotEmpty.notify_one(); } -void Pacer::renderFrame(std::shared_ptr& sceneRenderer, AVFrame* frame) -{ - // Count time spent in Pacer's queues - LARGE_INTEGER beforeRender, afterRender; - QueryPerformanceCounter(&beforeRender); - - // Render it - sceneRenderer->Render(frame); - QueryPerformanceCounter(&afterRender); - - if (frame->opaque_ref) { - auto* data = reinterpret_cast(frame->opaque_ref->data); - - m_DeviceResources->GetStats()->SubmitPacerTime( - beforeRender.QuadPart - data->decodeEndQpc, - afterRender.QuadPart - beforeRender.QuadPart); - } - - av_frame_free(&frame); - - // Drop frames if we have too many queued up for a while - std::unique_lock lock(m_FrameQueueLock); - - int frameDropTarget = m_FrameDropTarget; - for (int queueHistoryEntry : m_RenderQueueHistory) { - if (queueHistoryEntry == 0) { - // Be lenient as long as the queue length - // resolves before the end of frame history - frameDropTarget += 2; - break; - } - } - - // Keep a rolling 500 ms window of render queue history - if (m_RenderQueueHistory.size() == m_MaxVideoFps / 2) { - m_RenderQueueHistory.pop_front(); - } - - m_RenderQueueHistory.push_back((int)m_RenderQueue.size()); - - // Catch up if we're several frames ahead - int dropCount = 0; - while (m_RenderQueue.size() > frameDropTarget) { - ++dropCount; - AVFrame* frame = std::move(m_RenderQueue.front()); - m_RenderQueue.pop_front(); +void Pacer::frontPacer(std::shared_ptr &sceneRenderer, AVFrame *frame) { + // Count time spent in Pacer's queues + LARGE_INTEGER beforeRender, afterRender; + QueryPerformanceCounter(&beforeRender); + + // Render it + sceneRenderer->Render(frame); + QueryPerformanceCounter(&afterRender); + + if (frame->opaque_ref) { + auto *data = reinterpret_cast(frame->opaque_ref->data); + + m_DeviceResources->GetStats()->SubmitPacerTime( + beforeRender.QuadPart - data->decodeEndQpc, + afterRender.QuadPart - beforeRender.QuadPart); + } + + av_frame_free(&frame); + + // Drop frames if we have too many queued up for a while + std::unique_lock lock(m_FrameQueueLock); + + int frameDropTarget = m_FrameDropTarget; + for (int queueHistoryEntry : m_RenderQueueHistory) { + if (queueHistoryEntry == 0) { + // Be lenient as long as the queue length + // resolves before the end of frame history + frameDropTarget += 2; + break; + } + } + + // Keep a rolling 500 ms window of render queue history + if (m_RenderQueueHistory.size() == m_MaxVideoFps / 2) { + m_RenderQueueHistory.pop_front(); + } + + m_RenderQueueHistory.push_back((int)m_RenderQueue.size()); + + // Catch up if we're several frames ahead + int dropCount = 0; + while (m_RenderQueue.size() > frameDropTarget) { + ++dropCount; + AVFrame *frame = std::move(m_RenderQueue.front()); + m_RenderQueue.pop_front(); FQLog("! render queue overflow: %d > frame drop target %d, dropping oldest frame (pts %.3f ms)\n", - m_RenderQueue.size() + 1, frameDropTarget, frame->pts / 90.0); + m_RenderQueue.size() + 1, frameDropTarget, frame->pts / 90.0); m_DeviceResources->GetStats()->SubmitDroppedFrame(1); - // Drop the lock while we call av_frame_free() - lock.unlock(); - av_frame_free(&frame); - lock.lock(); - } + // Drop the lock while we call av_frame_free() + lock.unlock(); + av_frame_free(&frame); + lock.lock(); + } - lock.unlock(); + lock.unlock(); - // Don't graph 0 here, because it's already being done in dropFrameForEnqueue - if (dropCount > 0) { - ImGuiPlots::instance().observeFloat(PLOT_DROPPED_PACER_FRONT, (float)dropCount); - } + // Don't graph 0 here, because it's already being done in dropFrameForEnqueue + if (dropCount > 0) { + ImGuiPlots::instance().observeFloat(PLOT_DROPPED_PACER_FRONT, (float)dropCount); + } } -void Pacer::dropFrameForEnqueue(std::deque& queue, int plotId) -{ - int dropCount = 0; - if (queue.size() >= MAX_QUEUED_FRAMES) { - AVFrame* frame = std::move(queue.front()); - queue.pop_front(); +void Pacer::dropFrameForEnqueue(std::deque &queue, int plotId) { + int dropCount = 0; + if (queue.size() >= MAX_QUEUED_FRAMES) { + AVFrame *frame = std::move(queue.front()); + queue.pop_front(); - FQLog("! queue full (%d), dropping oldest frame (pts %.3f ms)\n", - queue.size() + 1, frame->pts / 90.0); + FQLog("! queue full (%d), dropping oldest frame (pts %.3f ms)\n", + queue.size() + 1, frame->pts / 90.0); m_DeviceResources->GetStats()->SubmitDroppedFrame(1); - av_frame_free(&frame); - dropCount = 1; - } + av_frame_free(&frame); + dropCount = 1; + } - ImGuiPlots::instance().observeFloat(plotId, (float)dropCount); + ImGuiPlots::instance().observeFloat(plotId, (float)dropCount); } -void Pacer::submitFrame(AVFrame* frame) -{ - // Make sure initialize() has been called - assert(m_MaxVideoFps != 0); - - if (m_UseVsyncThread) { - // Queue the frame and possibly wake up the render thread - { - std::scoped_lock lock(m_FrameQueueLock); - - dropFrameForEnqueue(m_PacingQueue, PLOT_DROPPED_PACER_BACK); - m_PacingQueue.push_back(frame); - - size_t queueSize = m_PacingQueue.size(); - ImGuiPlots::instance().observeFloat(PLOT_QUEUED_FRAMES, (float)queueSize); - } - - // notify pacing loop of new frame - m_PacingQueueNotEmpty.notify_one(); - } - else { - enqueueFrameForRenderingAndUnlock(frame); - } +void Pacer::submitFrame(AVFrame *frame) { + // Make sure initialize() has been called + assert(m_MaxVideoFps != 0); + + // Queue the frame and possibly wake up the render thread + { + std::scoped_lock lock(m_FrameQueueLock); + + dropFrameForEnqueue(m_PacingQueue, PLOT_DROPPED_PACER_BACK); + m_PacingQueue.push_back(frame); + } + + // notify pacing loop of new frame + FQLog("submitFrame notifying m_PacingQueueNotEmpty\n"); + m_PacingQueueNotEmpty.notify_one(); } int Pacer::getFrameDropTarget() { - return m_FrameDropTarget; + return m_FrameDropTarget; } int Pacer::modifyFrameDropTarget(bool increase) { - int current = m_FrameDropTarget; - if (increase) { - if (current >= 5) { - return current; - } - return ++m_FrameDropTarget; - } - else { - if (current == 0) { - return current; - } - return --m_FrameDropTarget; - } + int current = m_FrameDropTarget; + if (increase) { + if (current >= 5) { + return current; + } + return ++m_FrameDropTarget; + } else { + if (current == 0) { + return current; + } + return --m_FrameDropTarget; + } } diff --git a/Streaming/Pacer.h b/Streaming/Pacer.h index 2a2dde3f..626dd8e9 100644 --- a/Streaming/Pacer.h +++ b/Streaming/Pacer.h @@ -1,60 +1,66 @@ #pragma once -#include "VideoRenderer.h" -#include "Utils.hpp" #include #include #include +#include "Utils.hpp" +#include "VideoRenderer.h" -extern "C" -{ +extern "C" { #include } -class Pacer -{ -public: - Pacer(const std::shared_ptr& res); +class Pacer { + public: + Pacer(const std::shared_ptr &res); + + ~Pacer(); - ~Pacer(); + void submitFrame(AVFrame *frame); - void submitFrame(AVFrame* frame); + void initialize(int maxVideoFps, double refreshRate); - void initialize(int maxVideoFps, double refreshRate, bool useVsyncThread); + void waitForFrame(); - void waitForFrame(); + bool renderOnMainThread(std::shared_ptr &sceneRenderer); - bool renderOnMainThread(std::shared_ptr& sceneRenderer); + int getFrameDropTarget(); - int getFrameDropTarget(); + int modifyFrameDropTarget(bool increase); - int modifyFrameDropTarget(bool increase); + private: + void vsyncEmulator(); -private: - void vsyncThread(); + void backPacer(); - void waitForVsync(); + void handleVsync(int64_t nextDeadlineQpc); - void handleVsync(double timeUntilNextVsyncMillis); + void enqueueFrameForRenderingAndUnlock(AVFrame *frame); - void enqueueFrameForRenderingAndUnlock(AVFrame* frame); + void frontPacer(std::shared_ptr &sceneRenderer, AVFrame *frame); - void renderFrame(std::shared_ptr& sceneRenderer, AVFrame* frame); + void dropFrameForEnqueue(std::deque &queue, int plotId); - void dropFrameForEnqueue(std::deque& queue, int plotId); + bool stopping() const noexcept { + return m_Stopping.load(std::memory_order_acquire); + } - std::shared_ptr m_DeviceResources; - std::thread m_VsyncThread; - std::deque m_RenderQueue; - std::deque m_PacingQueue; - std::deque m_RenderQueueHistory; - std::deque m_PacingQueueHistory; - std::mutex m_FrameQueueLock; - std::condition_variable m_RenderQueueNotEmpty; - std::condition_variable m_PacingQueueNotEmpty; - std::atomic m_Stopping{false}; - int m_MaxVideoFps; - double m_DisplayFps; - bool m_UseVsyncThread; - std::atomic_int m_FrameDropTarget; + std::shared_ptr m_DeviceResources; + std::thread m_VsyncEmulator; + std::thread m_BackPacerThread; + std::mutex m_VsyncMutex; + std::condition_variable m_VsyncSignalled; + uint64_t m_VsyncSeq; + int64_t m_VsyncNextDeadlineQpc; + std::deque m_RenderQueue; + std::deque m_PacingQueue; + std::deque m_RenderQueueHistory; + std::deque m_PacingQueueHistory; + std::mutex m_FrameQueueLock; + std::condition_variable m_RenderQueueNotEmpty; + std::condition_variable m_PacingQueueNotEmpty; + std::atomic m_Stopping{false}; + int m_MaxVideoFps; + double m_RefreshRate; + std::atomic_int m_FrameDropTarget; }; diff --git a/Streaming/PacerRational.h b/Streaming/PacerRational.h new file mode 100644 index 00000000..843e9772 --- /dev/null +++ b/Streaming/PacerRational.h @@ -0,0 +1,59 @@ +// PacerRational.h +// Support methods for handling fractional refresh rates in Qpc integer base + +#pragma once +#include +#include +#include + +#pragma intrinsic(_umul128, _udiv128) + +// Represents a period in QPC ticks using an integer part plus a fractional accumulator. +// This avoids drift at fractional Hz such as 59.94 or 119.88. +struct QpcRationalPeriod { + int64_t baseTicks = 0; // Integer ticks added each step + uint64_t remainder = 0; // numerator for fractional tick + uint32_t den = 1; // denominator for fractional tick + uint64_t accum = 0; // accumulator for fractional carry + int64_t nextDeadlineQpc = 0; // absolute QPC for next tick + + // Initialize from exact Hz as a rational number: Hz = num/den. + // Period(QPC) = QpcFreq * den / num. + // Example: 59.94 Hz => num=60000, den=1001 + void initFromHz(uint32_t num, uint32_t hzDen, int64_t baseQpc) { + const uint64_t f = (uint64_t)QpcFreq(); + + uint64_t hi = 0; + uint64_t lo = _umul128(f, (uint64_t)hzDen, &hi); + uint64_t q = 0, r = 0; + div128by32(hi, lo, num, q, r); + + baseTicks = (int64_t)q; + remainder = r; + den = num; + accum = 0; + nextDeadlineQpc = baseQpc + baseTicks; + } + + // Advance to the next deadline + void step() { + nextDeadlineQpc += baseTicks; + if (remainder != 0) { + accum += remainder; + if (accum >= den) { + // Carry one extra QPC tick occasionally to account for the fractional period + nextDeadlineQpc += 1; + accum -= den; + } + } + } + + private: + // Divide a 128-bit {hi:lo} by 32-bit divisor + static void div128by32(uint64_t hi, uint64_t lo, + uint32_t divisor, + uint64_t "ient, uint64_t &remainder) { + // We can do this in two 64-bit steps with _udiv128 + quotient = _udiv128(hi, lo, divisor, &remainder); + } +}; diff --git a/Streaming/StatsRenderer.cpp b/Streaming/StatsRenderer.cpp index 1d7d00f9..0f22adbe 100644 --- a/Streaming/StatsRenderer.cpp +++ b/Streaming/StatsRenderer.cpp @@ -161,22 +161,26 @@ void StatsRenderer::RenderGraphs() ImGui::PopID(); }; - const int row1[3] = {PLOT_FRAMETIME, PLOT_DROPPED_NETWORK, PLOT_QUEUED_FRAMES}; + const int row1[3] = {PLOT_FRAMETIME, PLOT_DROPPED_NETWORK, PLOT_VSYNC_INTERVAL}; for (int c = 0; c < 3; ++c) { if (c > 0) ImGui::SameLine(0.0f, itemSpacingX); draw_plot(row1[c], graphW, graphH); } ImGui::Dummy(ImVec2(1.0f, itemSpacingY)); - const int row2[3] = {PLOT_HOST_FRAMETIME, PLOT_DROPPED_PACER_FRONT, PLOT_OVERHEAD}; - for (int c = 0; c < 3; ++c) { + const int row2[4] = {PLOT_HOST_FRAMETIME, PLOT_DROPPED_PACER_FRONT, PLOT_DROPPED_PACER_BACK, PLOT_OVERHEAD}; + for (int c = 0; c < 4; ++c) { + float width = graphW; + if (row2[c] == PLOT_DROPPED_PACER_FRONT || row2[c] == PLOT_DROPPED_PACER_BACK) { + width /= 2; + } if (c > 0) ImGui::SameLine(0.0f, itemSpacingX); - draw_plot(row2[c], graphW, graphH); + draw_plot(row2[c], width, graphH); } // 3rd row for quickly graphing something if needed - ImGui::Dummy(ImVec2(1.0f, itemSpacingY)); - draw_plot(PLOT_DROPPED_PACER_BACK, graphW, graphH); + // ImGui::Dummy(ImVec2(1.0f, itemSpacingY)); + // draw_plot(PLOT_DROPPED_PACER_BACK, graphW, graphH); ImGui::End(); diff --git a/Streaming/moonlight_xbox_dxMain.cpp b/Streaming/moonlight_xbox_dxMain.cpp index 42772617..ce20a4ff 100644 --- a/Streaming/moonlight_xbox_dxMain.cpp +++ b/Streaming/moonlight_xbox_dxMain.cpp @@ -108,6 +108,10 @@ void moonlight_xbox_dxMain::StartRenderLoop() // Create a task that will be run on a background thread. auto workItemHandler = ref new WorkItemHandler([this](IAsyncAction^ action) { + if (!SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_ABOVE_NORMAL)) { + Utils::Logf("Failed to set render thread priority: %d\n", GetLastError()); + } + // Calculate the updated frame and render once per vertical blanking interval. while (action->Status == AsyncStatus::Started) { @@ -135,7 +139,7 @@ void moonlight_xbox_dxMain::StartRenderLoop() } auto inputItemHandler = ref new WorkItemHandler([this](IAsyncAction^ action) { - // Target period = 1000hz + // Target period = 2000hz using clock = std::chrono::high_resolution_clock; const auto period = std::chrono::microseconds(2000); auto next = clock::now() + period; @@ -395,8 +399,10 @@ bool moonlight_xbox_dxMain::Render() // Render the scene objects. //LOCK_D3D("Render"); + FFMpegDecoder::instance().WaitForFrame(); bool shouldPresent = FFMpegDecoder::instance().RenderFrameOnMainThread(m_sceneRenderer); + m_LogRenderer->Render(); m_statsTextRenderer->Render(showImGui); //UNLOCK_D3D(); @@ -419,11 +425,9 @@ void moonlight_xbox_dxMain::Clear() // Clear the views. ID3D11RenderTargetView* renderTarget[] = { m_deviceResources->GetBackBufferRenderTargetView() }; - auto depthStencil = m_deviceResources->GetDepthStencilView(); context->ClearRenderTargetView(renderTarget[0], Colors::Black); - context->ClearDepthStencilView(depthStencil, D3D11_CLEAR_DEPTH | D3D11_CLEAR_STENCIL, 1.0f, 0); - context->OMSetRenderTargets(1, renderTarget, depthStencil); + context->OMSetRenderTargets(1, renderTarget, nullptr); // Set the viewport. const auto viewport = m_deviceResources->GetScreenViewport(); diff --git a/Streaming/moonlight_xbox_dxMain.h b/Streaming/moonlight_xbox_dxMain.h index cfb60a26..abd9ce0b 100644 --- a/Streaming/moonlight_xbox_dxMain.h +++ b/Streaming/moonlight_xbox_dxMain.h @@ -6,6 +6,10 @@ #include "Streaming\StatsRenderer.h" #include "Pages\StreamPage.xaml.h" +namespace DX { + class GpuPerformanceTimer; +} + // Renders Direct2D and 3D content on the screen. namespace moonlight_xbox_dx { diff --git a/moonlight-xbox-dx.vcxproj b/moonlight-xbox-dx.vcxproj index 78562272..19ad4cf6 100644 --- a/moonlight-xbox-dx.vcxproj +++ b/moonlight-xbox-dx.vcxproj @@ -347,6 +347,7 @@ + @@ -382,6 +383,7 @@ App.xaml + diff --git a/moonlight-xbox-dx.vcxproj.filters b/moonlight-xbox-dx.vcxproj.filters index 8ecfd554..b0d5eaa3 100644 --- a/moonlight-xbox-dx.vcxproj.filters +++ b/moonlight-xbox-dx.vcxproj.filters @@ -383,6 +383,9 @@ Source Files + + Source Files + @@ -472,6 +475,9 @@ Header Files + + Header Files + @@ -2968,6 +2974,230 @@ DLLs + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + DLLs + + + + DLLs + + + DLLs + diff --git a/pch.h b/pch.h index 491569be..afe813ff 100644 --- a/pch.h +++ b/pch.h @@ -43,34 +43,35 @@ ) // Time helpers -static int64_t QpcFreq() { - static LARGE_INTEGER qpcFreq = {0, 0}; - if (qpcFreq.QuadPart == 0) { - QueryPerformanceFrequency(&qpcFreq); - } - return qpcFreq.QuadPart; +static inline int64_t QpcFreq() { + static int64_t f = []{ + LARGE_INTEGER li{}; + QueryPerformanceFrequency(&li); + return li.QuadPart; + }(); + return f; } -static int64_t QpcToUs(int64_t qpc) { - int64_t qpcFreq = QpcFreq(); - return (qpc / qpcFreq) * UINT64_C(1000000) + - (qpc % qpcFreq) * UINT64_C(1000000) / qpcFreq; +static inline int64_t QpcNow() { + LARGE_INTEGER li{}; + QueryPerformanceCounter(&li); + return li.QuadPart; } -static int64_t QpcToNs(int64_t qpc) { - int64_t qpcFreq = QpcFreq(); - return (qpc / qpcFreq) * UINT64_C(1000000000) + - (qpc % qpcFreq) * UINT64_C(1000000000) / qpcFreq; +static inline int64_t UsToQpc(int64_t us) { + const int64_t f = QpcFreq(); + return (us / UINT64_C(1000000)) * f + + (us % UINT64_C(1000000)) * f / UINT64_C(1000000); } -static int64_t QpcNsNow() { - LARGE_INTEGER perf_count; - QueryPerformanceCounter(&perf_count); - return QpcToNs(perf_count.QuadPart); +static inline int64_t QpcToUs(int64_t qpc) { + const int64_t f = QpcFreq(); + return (qpc / f) * UINT64_C(1000000) + + (qpc % f) * UINT64_C(1000000) / f; } -static double QpcToMs(int64_t qpc) { - return (double)QpcToUs(qpc) / 1000.0f; +static inline double QpcToMs(int64_t qpc) { + return (double)QpcToUs(qpc) / 1000.0; } // Log something only once, safe to use in hot areas of the code @@ -86,7 +87,7 @@ static double QpcToMs(int64_t qpc) { // Frame queue debugging #if !defined(NDEBUG) -#define FRAME_QUEUE_VERBOSE +//#define FRAME_QUEUE_VERBOSE #endif #ifdef FRAME_QUEUE_VERBOSE From 13905e81d55bf79c7d2e283f274cfbad4fee6870 Mon Sep 17 00:00:00 2001 From: Andy Grundman Date: Sat, 27 Sep 2025 03:55:00 -0400 Subject: [PATCH 12/22] Fix several bugs related to the vsync host setting, it now works properly. Use only hardware vsync when streaming at <= 60fps. --- Common/DeviceResources.cpp | 22 +++++++- Common/DeviceResources.h | 2 + Pages/StreamPage.xaml.cpp | 1 + Plot/PlotDesc.h | 2 +- State/MoonlightClient.cpp | 22 ++++++-- State/MoonlightHost.h | 2 +- Streaming/Pacer.cpp | 107 ++++++++++++++++++++++++++++++------- Streaming/Pacer.h | 6 ++- Streaming/PacerRational.h | 18 ++----- 9 files changed, 138 insertions(+), 44 deletions(-) diff --git a/Common/DeviceResources.cpp b/Common/DeviceResources.cpp index 03d0e52a..21117a17 100644 --- a/Common/DeviceResources.cpp +++ b/Common/DeviceResources.cpp @@ -241,13 +241,13 @@ void DX::DeviceResources::CreateWindowSizeDependentResources() m_d3dRenderTargetSize.Width = normalizedWidth; m_d3dRenderTargetSize.Height = normalizedHeight; - if (m_swapChain != nullptr) + if (m_swapChain != nullptr && m_enableVsync == m_swapchainVsync) { // If the swap chain already exists, resize it. int flags = DXGI_SWAP_CHAIN_FLAG_FRAME_LATENCY_WAITABLE_OBJECT; if (!m_enableVsync) flags |= DXGI_SWAP_CHAIN_FLAG_ALLOW_TEARING; HRESULT hr = m_swapChain->ResizeBuffers( - 2, // Double-buffered swap chain. + 0, // Don't change the number of buffers lround(m_d3dRenderTargetSize.Width), lround(m_d3dRenderTargetSize.Height), m_backBufferFormat, @@ -511,6 +511,11 @@ void DX::DeviceResources::SetCompositionScale(float compositionScaleX, float com } } +void DX::DeviceResources::SetVsync(bool enableVsync) +{ + m_enableVsync = enableVsync; +} + // This method is called in the event handler for the DisplayContentsInvalidated event. void DX::DeviceResources::ValidateDevice() { @@ -716,6 +721,19 @@ bool is_running_on_xbox(void) return (device_family == L"Windows.Xbox"); } +bool DX::DeviceResources::isXbox() +{ + GAMING_DEVICE_MODEL_INFORMATION info; + if (FAILED(GetGamingDeviceModelInformation(&info))) { + return false; + } + + if (info.vendorId == GAMING_DEVICE_VENDOR_ID_MICROSOFT) { + return true; + } + return false; +} + int DX::DeviceResources::uwp_get_height() { if (is_running_on_xbox()) diff --git a/Common/DeviceResources.h b/Common/DeviceResources.h index 6181e593..6b511045 100644 --- a/Common/DeviceResources.h +++ b/Common/DeviceResources.h @@ -22,6 +22,7 @@ namespace DX void SetCurrentOrientation(Windows::Graphics::Display::DisplayOrientations currentOrientation); void SetDpi(float dpi); void SetCompositionScale(float compositionScaleX, float compositionScaleY); + void SetVsync(bool enableVsync); void ValidateDevice(); void HandleDeviceLost(); void RegisterDeviceNotify(IDeviceNotify* deviceNotify); @@ -30,6 +31,7 @@ namespace DX void WaitOnSwapChain(); void GetUWPPixelDimensions(uint32_t *width, uint32_t *height); double GetUWPRefreshRate(); + bool isXbox(); static int uwp_get_width(); static int uwp_get_height(); diff --git a/Pages/StreamPage.xaml.cpp b/Pages/StreamPage.xaml.cpp index 16600076..9efc3dbc 100644 --- a/Pages/StreamPage.xaml.cpp +++ b/Pages/StreamPage.xaml.cpp @@ -142,6 +142,7 @@ void StreamPage::toggleLogsButton_Click(Platform::Object^ sender, Windows::UI::X void StreamPage::OnNavigatedTo(Windows::UI::Xaml::Navigation::NavigationEventArgs^ e) { configuration = dynamic_cast(e->Parameter); SetStreamConfig(configuration); + m_deviceResources->SetVsync(configuration->enableVsync); if (configuration == nullptr)return; } diff --git a/Plot/PlotDesc.h b/Plot/PlotDesc.h index 62a60eec..e80f2f0b 100644 --- a/Plot/PlotDesc.h +++ b/Plot/PlotDesc.h @@ -38,7 +38,7 @@ struct PlotDesc inline constexpr std::array kPlotDescs = {{ {"Frametime", PLOT_LABEL_MIN_MAX_AVG, "ms", -0.1f, 50.0f, 0.0f, 49.9f}, {"Host Frametime", PLOT_LABEL_MIN_MAX_AVG, "ms", -0.1f, 50.0f, 0.0f, 49.9f}, - {"Vsync interval", PLOT_LABEL_MIN_MAX_AVG, "ms", -0.1f, 20.0f, 0.0f, 0.0f}, + {"Vsync interval", PLOT_LABEL_MIN_MAX_AVG, "ms", -0.1f, 25.0f, 0.0f, 0.0f}, {"Dropped Frames (network)", PLOT_LABEL_TOTAL_INT, "", -1.0f, 3.0f, 0.0f, 0.0f}, {"Dropped Frames (back pacing)", PLOT_LABEL_TOTAL_INT, "", -1.0f, 3.0f, 0.0f, 0.0f}, {"Dropped Frames (front pacing)", PLOT_LABEL_TOTAL_INT, "", -1.0f, 3.0f, 0.0f, 0.0f}, diff --git a/State/MoonlightClient.cpp b/State/MoonlightClient.cpp index 6b504920..1c10b59e 100644 --- a/State/MoonlightClient.cpp +++ b/State/MoonlightClient.cpp @@ -218,15 +218,27 @@ int MoonlightClient::StartStreaming(std::shared_ptr res, St config.width = sConfig->width; config.height = sConfig->height; config.bitrate = sConfig->bitrate; - if (res->GetRefreshRate() > 0.0) { - // request stream matching our exact fractional refresh rate - config.clientRefreshRateX100 = (int)(res->GetRefreshRate() * 100.0); - Utils::Logf("Requesting stream with clientRefreshRateX100=%d for %.2f\n", config.clientRefreshRateX100, res->GetRefreshRate()); + config.fps = sConfig->FPS; + if (res->GetRefreshRate() > 0.0 && info.vendorId == GAMING_DEVICE_VENDOR_ID_MICROSOFT) { + // Pass fractional refresh rate to host in case it's supported + switch (config.fps) { + case 120: + config.clientRefreshRateX100 = (int)(res->GetRefreshRate() * 100.0); + break; + case 60: + config.clientRefreshRateX100 = 5994; // enable 60fps stream in 120hz + break; + default: + config.clientRefreshRateX100 = sConfig->FPS * 100; + break; + } + + Utils::Logf("Requesting stream with clientRefreshRateX100=%d for %d FPS within %.2f Hz\n", + config.clientRefreshRateX100, config.fps, res->GetRefreshRate()); } config.colorRange = this->IsRGBFull() ? COLOR_RANGE_FULL : COLOR_RANGE_LIMITED; config.colorSpace = COLORSPACE_REC_601; config.encryptionFlags = 0; - config.fps = sConfig->FPS; config.packetSize = 1024; config.supportedVideoFormats = VIDEO_FORMAT_H264; if (!isXboxOne) { diff --git a/State/MoonlightHost.h b/State/MoonlightHost.h index f214e3af..b617e416 100644 --- a/State/MoonlightHost.h +++ b/State/MoonlightHost.h @@ -29,7 +29,7 @@ namespace moonlight_xbox_dx { bool enableSOPS = false; bool enableStats = false; bool enableGraphs = true; - bool enableVsync = true; + bool enableVsync = false; Windows::Foundation::Collections::IVector^ apps; public: //Thanks to https://phsucharee.wordpress.com/2013/06/19/data-binding-and-ccx-inotifypropertychanged/ diff --git a/Streaming/Pacer.cpp b/Streaming/Pacer.cpp index a3144b07..fafd6e68 100644 --- a/Streaming/Pacer.cpp +++ b/Streaming/Pacer.cpp @@ -61,7 +61,7 @@ Pacer::Pacer(const std::shared_ptr &res) : m_VsyncSeq(0), m_VsyncNextDeadlineQpc(0), m_Stopping(false), - m_MaxVideoFps(0), + m_StreamFps(0), m_RefreshRate(0.0), m_FrameDropTarget(0) { } @@ -101,16 +101,24 @@ Pacer::~Pacer() { } } -void Pacer::initialize(int maxVideoFps, double refreshRate) { - m_MaxVideoFps = maxVideoFps; +void Pacer::initialize(int streamFps, double refreshRate) { + m_StreamFps = streamFps; m_RefreshRate = refreshRate; Utils::Logf("Frame pacing: target %.2f Hz with %d FPS stream", - m_RefreshRate, m_MaxVideoFps); + m_RefreshRate, m_StreamFps); - // Start the timing emulator thread - if (!m_VsyncEmulator.joinable()) { - m_VsyncEmulator = std::thread(&Pacer::vsyncEmulator, this); + // If running on Xbox at 120hz, emulate the vsync interval + if (m_DeviceResources->isXbox() && m_StreamFps == 120) { + // Start the timing emulator thread + if (!m_VsyncEmulator.joinable()) { + m_VsyncEmulator = std::thread(&Pacer::vsyncEmulator, this); + } + } else { + // Much simpler hardware vsync thread + if (!m_VsyncEmulator.joinable()) { + m_VsyncEmulator = std::thread(&Pacer::vsyncHardware, this); + } } // Start the dedicated vblank pacing thread @@ -156,6 +164,24 @@ static inline void SleepUntilQpc(int64_t targetQpc, } } +int64_t Pacer::measureVsyncQpc(int intervals) { + auto *out = m_DeviceResources->GetDXGIOutput(); + + // Align to a vblank edge and measure deltas + out->WaitForVBlank(); + int64_t prev = QpcNow(); + int64_t sum = 0; + + for (int i = 0; i < intervals; ++i) { + out->WaitForVBlank(); + const int64_t now = QpcNow(); + sum += (now - prev); + prev = now; + } + + return sum / intervals; +} + void Pacer::vsyncEmulator() { if (!SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_HIGHEST)) { Utils::Logf("Failed to set vsyncEmulator priority: %d\n", GetLastError()); @@ -184,27 +210,41 @@ void Pacer::vsyncEmulator() { break; } - Utils::Logf("Pacer: emulating vsync for %.2fhz using integer rate %lu/%lu\n", + // measure system vsync rate so we stay in phase + // I experimented with having this run at 240hz but it doesn't seem necessary + int64_t systemVsyncQpc = measureVsyncQpc(8); + int32_t phase = (int32_t)(QpcToMs(systemVsyncQpc) / 1000.0 / (1.0 / ((double)hzNum / hzDen)) + 0.5); // e.g. 16.6 / 8.3 = 2 + + Utils::Logf("Pacer: hardware vsync interval %.2f ms\n", QpcToMs(systemVsyncQpc)); + Utils::Logf("Pacer: using emulated vsync timer @ %.2f Hz using integer rate %lu/%lu\n", m_RefreshRate, hzNum, hzDen); - static constexpr int64_t remeasureEveryFrames = 3600; // 30s at 120hz + // 30s resync seems about right, on Series X it drifts about 0.3ms in this time + static constexpr int32_t remeasureEveryFrames = 120 * 30; uint64_t frames = 0; int64_t baseQpc = 0; - int64_t lastVsync = 0; + int32_t resyncInFrames = 0; while (!stopping()) { int64_t curDeadlineQpc = period.nextDeadlineQpc; - if (baseQpc == 0 || frames % remeasureEveryFrames == 0) { + if (resyncInFrames <= 0 && frames % phase == 0) { // Align to a real vblank edge at startup and at regular intervals m_DeviceResources->GetDXGIOutput()->WaitForVBlank(); baseQpc = QpcNow(); period.initFromHz(hzNum, hzDen, baseQpc); + if (curDeadlineQpc > 0) { + Utils::Logf("resync to hardware vsync, frame: %d, drift: %lld ticks (%.3f ms), phase: %d\n", + frames, period.nextDeadlineQpc - curDeadlineQpc, QpcToMs(period.nextDeadlineQpc - curDeadlineQpc), phase); + } + curDeadlineQpc = baseQpc; + resyncInFrames = remeasureEveryFrames; } else { // Emulate the vsync interval SleepUntilQpc(period.nextDeadlineQpc); + --resyncInFrames; } // Compute next period @@ -223,9 +263,41 @@ void Pacer::vsyncEmulator() { } } +void Pacer::vsyncHardware() { + if (!SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_ABOVE_NORMAL)) { + Utils::Logf("Failed to set vsyncHardware priority: %d\n", GetLastError()); + } + + // measure system vsync rate + int64_t systemVsyncQpc = measureVsyncQpc(8); + int64_t lastT0 = 0; + + Utils::Logf("Pacer: using hardware vsync interval %.2f ms\n", QpcToMs(systemVsyncQpc)); + + while (!stopping()) { + // Align to a real vblank + m_DeviceResources->GetDXGIOutput()->WaitForVBlank(); + const int64_t now = QpcNow(); + + // signal vsync thread, passing it the next deadline + { + std::lock_guard lock(m_VsyncMutex); + ++m_VsyncSeq; + m_VsyncNextDeadlineQpc = now + systemVsyncQpc; + } + m_VsyncSignalled.notify_one(); + + if (lastT0 > 0) { + ImGuiPlots::instance().observeFloat(PLOT_VSYNC_INTERVAL, (float)QpcToMs(now - lastT0)); + } + lastT0 = now; + } + +} + void Pacer::backPacer() { if (!SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_ABOVE_NORMAL)) { - Utils::Logf("Failed to set vsyncThread priority: %d\n", GetLastError()); + Utils::Logf("Failed to set backPacer priority: %d\n", GetLastError()); } uint64_t seenSeq = 0; @@ -251,12 +323,11 @@ void Pacer::backPacer() { LARGE_INTEGER t0; static int64_t lastT0 = 0; QueryPerformanceCounter(&t0); - FQLog("vsyncThread interval %.3f ms\n", QpcToMs(t0.QuadPart - lastT0)); + FQLog("backPacer interval %.3f ms\n", QpcToMs(t0.QuadPart - lastT0)); lastT0 = t0.QuadPart; } } -// XXX rename this void Pacer::handleVsync(int64_t nextDeadlineQpc) { // Compute remaining time until the next edge, then subtract slack. const int64_t now = QpcNow(); @@ -276,8 +347,7 @@ void Pacer::handleVsync(int64_t nextDeadlineQpc) { // If we may get more frames per second than we can display, use // frame history to drop frames only if consistently above the // one queued frame mark. - // XXX this should match for 119.88 now - if (m_MaxVideoFps >= m_RefreshRate) { + if (m_StreamFps >= std::round(m_RefreshRate)) { for (int queueHistoryEntry : m_PacingQueueHistory) { if (queueHistoryEntry <= 1) { // Be lenient as long as the queue length @@ -432,7 +502,7 @@ void Pacer::frontPacer(std::shared_ptr &sceneRenderer, AVFrame *f } // Keep a rolling 500 ms window of render queue history - if (m_RenderQueueHistory.size() == m_MaxVideoFps / 2) { + if (m_RenderQueueHistory.size() == m_StreamFps / 2) { m_RenderQueueHistory.pop_front(); } @@ -481,9 +551,6 @@ void Pacer::dropFrameForEnqueue(std::deque &queue, int plotId) { } void Pacer::submitFrame(AVFrame *frame) { - // Make sure initialize() has been called - assert(m_MaxVideoFps != 0); - // Queue the frame and possibly wake up the render thread { std::scoped_lock lock(m_FrameQueueLock); diff --git a/Streaming/Pacer.h b/Streaming/Pacer.h index 626dd8e9..60223a1c 100644 --- a/Streaming/Pacer.h +++ b/Streaming/Pacer.h @@ -29,8 +29,12 @@ class Pacer { int modifyFrameDropTarget(bool increase); private: + int64_t measureVsyncQpc(int intervals); + void vsyncEmulator(); + void vsyncHardware(); + void backPacer(); void handleVsync(int64_t nextDeadlineQpc); @@ -60,7 +64,7 @@ class Pacer { std::condition_variable m_RenderQueueNotEmpty; std::condition_variable m_PacingQueueNotEmpty; std::atomic m_Stopping{false}; - int m_MaxVideoFps; + int m_StreamFps; double m_RefreshRate; std::atomic_int m_FrameDropTarget; }; diff --git a/Streaming/PacerRational.h b/Streaming/PacerRational.h index 843e9772..4d1a90dc 100644 --- a/Streaming/PacerRational.h +++ b/Streaming/PacerRational.h @@ -19,18 +19,17 @@ struct QpcRationalPeriod { // Initialize from exact Hz as a rational number: Hz = num/den. // Period(QPC) = QpcFreq * den / num. - // Example: 59.94 Hz => num=60000, den=1001 - void initFromHz(uint32_t num, uint32_t hzDen, int64_t baseQpc) { + void initFromHz(uint32_t hzNum, uint32_t hzDen, int64_t baseQpc) { const uint64_t f = (uint64_t)QpcFreq(); uint64_t hi = 0; uint64_t lo = _umul128(f, (uint64_t)hzDen, &hi); - uint64_t q = 0, r = 0; - div128by32(hi, lo, num, q, r); + uint64_t r = 0; + uint64_t q = _udiv128(hi, lo, hzNum, &r); baseTicks = (int64_t)q; remainder = r; - den = num; + den = hzNum; accum = 0; nextDeadlineQpc = baseQpc + baseTicks; } @@ -47,13 +46,4 @@ struct QpcRationalPeriod { } } } - - private: - // Divide a 128-bit {hi:lo} by 32-bit divisor - static void div128by32(uint64_t hi, uint64_t lo, - uint32_t divisor, - uint64_t "ient, uint64_t &remainder) { - // We can do this in two 64-bit steps with _udiv128 - quotient = _udiv128(hi, lo, divisor, &remainder); - } }; From 14d1180804cb772fe6297ebdfb3600f7a23225cc Mon Sep 17 00:00:00 2001 From: Andy Grundman Date: Sat, 27 Sep 2025 19:15:08 -0400 Subject: [PATCH 13/22] Use 10-bit offsets in BT2020 shader --- Assets/Shader/d3d11_bt2020lim_pixel.fxc | Bin 1340 -> 1340 bytes Assets/Shader/d3d11_bt2020lim_pixel.hlsl | 4 ++-- Streaming/VideoRenderer.cpp | 14 +++++++++++--- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/Assets/Shader/d3d11_bt2020lim_pixel.fxc b/Assets/Shader/d3d11_bt2020lim_pixel.fxc index d557a9edf09d6e7b1d6de091587613464411d27c..eeda4171205d1e673dbf36431face60bf9d4517d 100644 GIT binary patch delta 55 zcmdnPwTDZ@CBn(szu}}c|8a|-KZ+_O4@%8unkbUN#-Y%#mqTG=(>F#g2$Nwyhr;G2 Hrd><`IRO(& delta 55 zcmdnPwTDZ@CBn)1n(_3=razqr&hZ{}=gl~xG*Kjjt+AnDZ)3y8rf-Z~5GKR^#)i#J HOuLu>nJ5@I diff --git a/Assets/Shader/d3d11_bt2020lim_pixel.hlsl b/Assets/Shader/d3d11_bt2020lim_pixel.hlsl index e4e7bc40..4b90b692 100644 --- a/Assets/Shader/d3d11_bt2020lim_pixel.hlsl +++ b/Assets/Shader/d3d11_bt2020lim_pixel.hlsl @@ -9,7 +9,7 @@ static const min16float3x3 cscMatrix = static const min16float3 offsets = { - 16.0 / 255.0, 128.0 / 255.0, 128.0 / 255.0 + 64.0 / 1023.0, 512.0 / 1023.0, 512.0 / 1023.0 }; -#include "d3d11_video_pixel_end.hlsli" \ No newline at end of file +#include "d3d11_video_pixel_end.hlsli" diff --git a/Streaming/VideoRenderer.cpp b/Streaming/VideoRenderer.cpp index 490d9d6c..e11387ff 100644 --- a/Streaming/VideoRenderer.cpp +++ b/Streaming/VideoRenderer.cpp @@ -58,8 +58,12 @@ static const float k_CscMatrix_Bt2020Full[CSC_MATRIX_RAW_ELEMENT_COUNT] = { #define OFFSETS_ELEMENT_COUNT 3 +// 8-bit offsets static const float k_Offsets_Lim[OFFSETS_ELEMENT_COUNT] = { 16.0f / 255.0f, 128.0f / 255.0f, 128.0f / 255.0f }; static const float k_Offsets_Full[OFFSETS_ELEMENT_COUNT] = { 0.0f, 128.0f / 255.0f, 128.0f / 255.0f }; +// 10-bit offsets +static const float k_Offsets_10bit_Lim[OFFSETS_ELEMENT_COUNT] = { 64.0f / 1023.0f, 512.0f / 1023.0f, 512.0f / 1023.0f }; +static const float k_Offsets_10bit_Full[OFFSETS_ELEMENT_COUNT] = { 0.0f, 512.0f / 1023.0f, 512.0f / 1023.0f }; typedef struct _CSC_CONST_BUF { @@ -528,9 +532,13 @@ void VideoRenderer::bindColorConversion(AVFrame* frame) // No adjustments are needed to the float[3] array of offsets, so it can just // be copied with memcpy(). - memcpy(constBuf.offsets, - fullRange ? k_Offsets_Full : k_Offsets_Lim, - sizeof(constBuf.offsets)); + if (colorspace == COLORSPACE_REC_2020) { + // This is important to avoid tinting the image slightly + memcpy(constBuf.offsets, fullRange ? k_Offsets_10bit_Full : k_Offsets_10bit_Lim, sizeof(constBuf.offsets)); + } + else { + memcpy(constBuf.offsets, fullRange ? k_Offsets_Full : k_Offsets_Lim, sizeof(constBuf.offsets)); + } D3D11_SUBRESOURCE_DATA constData = {}; constData.pSysMem = &constBuf; From 8f79b2f0d4f74e7ef275467d65149fb33ee04c3b Mon Sep 17 00:00:00 2001 From: Andy Grundman Date: Wed, 1 Oct 2025 17:49:40 -0400 Subject: [PATCH 14/22] Misc cleanup and optimizations. Avoid creating render texture and SRVs every frame. --- Common/DeviceResources.cpp | 106 +-------------- Common/DeviceResources.h | 1 - Streaming/VideoRenderer.cpp | 192 +++++++++++++--------------- Streaming/VideoRenderer.h | 9 +- Streaming/moonlight_xbox_dxMain.cpp | 14 +- 5 files changed, 102 insertions(+), 220 deletions(-) diff --git a/Common/DeviceResources.cpp b/Common/DeviceResources.cpp index 21117a17..bc6e274b 100644 --- a/Common/DeviceResources.cpp +++ b/Common/DeviceResources.cpp @@ -229,12 +229,6 @@ void DX::DeviceResources::CreateWindowSizeDependentResources() UpdateRenderTargetSize(); - // The width and height of the swap chain must be based on the window's - // natively-oriented width and height. If the window is not in the native - // orientation, the dimensions must be reversed. - DXGI_MODE_ROTATION displayRotation = ComputeDisplayRotation(); - - bool swapDimensions = displayRotation == DXGI_MODE_ROTATION_ROTATE90 || displayRotation == DXGI_MODE_ROTATION_ROTATE270; //Get the correct screen resolution and adapt the swapchain to 16:9 aspect ratio float normalizedWidth = uwp_get_width(); float normalizedHeight = uwp_get_height(); @@ -347,48 +341,6 @@ void DX::DeviceResources::CreateWindowSizeDependentResources() // the swapchain. m_swapchainVsync = m_enableVsync; - // Set the proper orientation for the swap chain, and generate 2D and - // 3D matrix transformations for rendering to the rotated swap chain. - // Note the rotation angle for the 2D and 3D transforms are different. - // This is due to the difference in coordinate spaces. Additionally, - // the 3D matrix is specified explicitly to avoid rounding errors. - - switch (displayRotation) - { - case DXGI_MODE_ROTATION_IDENTITY: - m_orientationTransform2D = Matrix3x2F::Identity(); - m_orientationTransform3D = ScreenRotation::Rotation0; - break; - - case DXGI_MODE_ROTATION_ROTATE90: - m_orientationTransform2D = - Matrix3x2F::Rotation(90.0f) * - Matrix3x2F::Translation(m_logicalSize.Height, 0.0f); - m_orientationTransform3D = ScreenRotation::Rotation270; - break; - - case DXGI_MODE_ROTATION_ROTATE180: - m_orientationTransform2D = - Matrix3x2F::Rotation(180.0f) * - Matrix3x2F::Translation(m_logicalSize.Width, m_logicalSize.Height); - m_orientationTransform3D = ScreenRotation::Rotation180; - break; - - case DXGI_MODE_ROTATION_ROTATE270: - m_orientationTransform2D = - Matrix3x2F::Rotation(270.0f) * - Matrix3x2F::Translation(0.0f, m_logicalSize.Width); - m_orientationTransform3D = ScreenRotation::Rotation90; - break; - - default: - throw ref new FailureException(); - } - - DX::ThrowIfFailed( - m_swapChain->SetRotation(displayRotation) - ); - // Setup inverse scale on the swap chain DXGI_MATRIX_3X2_F inverseScale = { 0 }; inverseScale._11 = 1.0f / m_effectiveCompositionScaleX; @@ -485,6 +437,7 @@ void DX::DeviceResources::SetDpi(float dpi) { if (dpi != m_dpi) { + m_dpi = dpi; CreateWindowSizeDependentResources(); } } @@ -616,7 +569,7 @@ void DX::DeviceResources::Present() //LOCK_D3D("Present"); if (m_enableVsync) { - // This will still use vsync due to the lack of DXGI_PRESENT_ALLOW_TEARING + // Composition swapchain hr = m_swapChain->Present1(0, 0, ¶meters); } else { @@ -658,61 +611,6 @@ void DX::DeviceResources::Present() } } -// This method determines the rotation between the display device's native orientation and the -// current display orientation. -DXGI_MODE_ROTATION DX::DeviceResources::ComputeDisplayRotation() -{ - DXGI_MODE_ROTATION rotation = DXGI_MODE_ROTATION_UNSPECIFIED; - - // Note: NativeOrientation can only be Landscape or Portrait even though - // the DisplayOrientations enum has other values. - switch (m_nativeOrientation) - { - case DisplayOrientations::Landscape: - switch (m_currentOrientation) - { - case DisplayOrientations::Landscape: - rotation = DXGI_MODE_ROTATION_IDENTITY; - break; - - case DisplayOrientations::Portrait: - rotation = DXGI_MODE_ROTATION_ROTATE270; - break; - - case DisplayOrientations::LandscapeFlipped: - rotation = DXGI_MODE_ROTATION_ROTATE180; - break; - - case DisplayOrientations::PortraitFlipped: - rotation = DXGI_MODE_ROTATION_ROTATE90; - break; - } - break; - - case DisplayOrientations::Portrait: - switch (m_currentOrientation) - { - case DisplayOrientations::Landscape: - rotation = DXGI_MODE_ROTATION_ROTATE90; - break; - - case DisplayOrientations::Portrait: - rotation = DXGI_MODE_ROTATION_IDENTITY; - break; - - case DisplayOrientations::LandscapeFlipped: - rotation = DXGI_MODE_ROTATION_ROTATE270; - break; - - case DisplayOrientations::PortraitFlipped: - rotation = DXGI_MODE_ROTATION_ROTATE180; - break; - } - break; - } - return rotation; -} - //Thank you tunip3 for https://github.com/libretro/RetroArch/pull/13406/files //Check if we are running on Xbox bool is_running_on_xbox(void) diff --git a/Common/DeviceResources.h b/Common/DeviceResources.h index 6b511045..86164e77 100644 --- a/Common/DeviceResources.h +++ b/Common/DeviceResources.h @@ -78,7 +78,6 @@ namespace DX void CreateDeviceResources(); void CreateWindowSizeDependentResources(); void UpdateRenderTargetSize(); - DXGI_MODE_ROTATION ComputeDisplayRotation(); void ImGui_Init(const Microsoft::WRL::ComPtr& panelNative, float dpi); void ImGui_Deinit(); diff --git a/Streaming/VideoRenderer.cpp b/Streaming/VideoRenderer.cpp index e11387ff..4d9ae2d0 100644 --- a/Streaming/VideoRenderer.cpp +++ b/Streaming/VideoRenderer.cpp @@ -103,117 +103,117 @@ void VideoRenderer::Update(DX::StepTimer const& timer) } +void VideoRenderer::EnsureYuvTargets(ID3D11Device* dev, DXGI_FORMAT fmt, UINT w, UINT h) { + if (m_yuvTexture && m_yuvFormat == fmt && m_yuvWidth == w && m_yuvHeight == h) + return; + + m_yuvTexture.Reset(); + m_srvY.Reset(); + m_srvUV.Reset(); + + D3D11_TEXTURE2D_DESC d = {}; + d.Width = w; + d.Height = h; + d.MipLevels = 1; + d.ArraySize = 1; + d.Format = fmt; + d.SampleDesc.Count = 1; + d.Usage = D3D11_USAGE_DEFAULT; + d.BindFlags = D3D11_BIND_SHADER_RESOURCE; + DX::ThrowIfFailed(dev->CreateTexture2D(&d, nullptr, &m_yuvTexture)); + + D3D11_SHADER_RESOURCE_VIEW_DESC yDesc = {}; + yDesc.Format = (fmt == DXGI_FORMAT_P010) ? DXGI_FORMAT_R16_UNORM : DXGI_FORMAT_R8_UNORM; + yDesc.ViewDimension = D3D11_SRV_DIMENSION_TEXTURE2D; + yDesc.Texture2D.MostDetailedMip = 0; + yDesc.Texture2D.MipLevels = 1; + DX::ThrowIfFailed(dev->CreateShaderResourceView(m_yuvTexture.Get(), &yDesc, &m_srvY)); + + D3D11_SHADER_RESOURCE_VIEW_DESC uvDesc = {}; + uvDesc.Format = (fmt == DXGI_FORMAT_P010) ? DXGI_FORMAT_R16G16_UNORM : DXGI_FORMAT_R8G8_UNORM; + uvDesc.ViewDimension = D3D11_SRV_DIMENSION_TEXTURE2D; + uvDesc.Texture2D.MostDetailedMip = 0; + uvDesc.Texture2D.MipLevels = 1; + DX::ThrowIfFailed(dev->CreateShaderResourceView(m_yuvTexture.Get(), &uvDesc, &m_srvUV)); + + m_yuvFormat = fmt; + m_yuvWidth = w; + m_yuvHeight = h; +} + bool renderedOneFrame = false; // Renders one frame using the vertex and pixel shaders. -bool VideoRenderer::Render(AVFrame* frame) -{ +bool VideoRenderer::Render(AVFrame *frame) { // Loading is asynchronous. Only draw geometry after it's loaded. - if (!m_loadingComplete) - { + if (!m_loadingComplete.load(std::memory_order_acquire)) { return true; } - //LOCK_D3D("VideoRenderer->Render"); + // LOCK_D3D("VideoRenderer->Render"); - auto d3dcontext = m_deviceResources->GetD3DDeviceContext(); - auto d3ddevice = m_deviceResources->GetD3DDevice(); - - Microsoft::WRL::ComPtr renderTexture; - D3D11_BOX box; - box.left = 0; - box.top = 0; - box.right = renderTextureDesc.Width; - box.bottom = renderTextureDesc.Height; - box.front = 0; - box.back = 1; + auto *ctx = m_deviceResources->GetD3DDeviceContext(); + auto *dev = m_deviceResources->GetD3DDevice(); - ID3D11Texture2D* ffmpegTexture = (ID3D11Texture2D*)(frame->data[0]); + ID3D11Texture2D *ffmpegTexture = (ID3D11Texture2D *)(frame->data[0]); D3D11_TEXTURE2D_DESC ffmpegDesc; ffmpegTexture->GetDesc(&ffmpegDesc); int index = (int)(frame->data[1]); - box.right = std::min(renderTextureDesc.Width, ffmpegDesc.Width); - box.bottom = std::min(renderTextureDesc.Height, ffmpegDesc.Height); - renderTextureDesc.Format = ffmpegDesc.Format; - DX::ThrowIfFailed(d3ddevice->CreateTexture2D(&renderTextureDesc, NULL, renderTexture.GetAddressOf()), "Render Texture Creation"); - d3dcontext->CopySubresourceRegion(renderTexture.Get(), 0, 0, 0, 0, ffmpegTexture, index, &box); + + EnsureYuvTargets(dev, ffmpegDesc.Format, frame->width, frame->height); + + ID3D11ShaderResourceView *nullSrvs[2] = {nullptr, nullptr}; + ctx->PSSetShaderResources(0, 2, nullSrvs); + + D3D11_BOX box{}; + box.right = frame->width; + box.bottom = frame->height; + box.back = 1; + ctx->CopySubresourceRegion(m_yuvTexture.Get(), 0, 0, 0, 0, ffmpegTexture, index, &box); + + ID3D11ShaderResourceView *srvs[2] = {m_srvY.Get(), m_srvUV.Get()}; + ctx->PSSetShaderResources(0, 2, srvs); UINT stride = sizeof(VERTEX); UINT offset = 0; - d3dcontext->IASetIndexBuffer(m_indexBuffer.Get(), - DXGI_FORMAT_R32_UINT, - 0); - d3dcontext->IASetVertexBuffers( - 0, - 1, - m_vertexBuffer.GetAddressOf(), - &stride, - &offset - ); - d3dcontext->IASetPrimitiveTopology(D3D11_PRIMITIVE_TOPOLOGY_TRIANGLELIST); - d3dcontext->IASetInputLayout(m_inputLayout.Get()); + ctx->IASetIndexBuffer(m_indexBuffer.Get(), DXGI_FORMAT_R32_UINT, 0); + ctx->IASetVertexBuffers(0, 1, m_vertexBuffer.GetAddressOf(), &stride, &offset); + ctx->IASetPrimitiveTopology(D3D11_PRIMITIVE_TOPOLOGY_TRIANGLELIST); + ctx->IASetInputLayout(m_inputLayout.Get()); // Attach our vertex shader. - d3dcontext->VSSetShader(m_vertexShader.Get(), nullptr, 0); - - Microsoft::WRL::ComPtr m_luminance_shader_resource_view; - Microsoft::WRL::ComPtr m_chrominance_shader_resource_view; - D3D11_SHADER_RESOURCE_VIEW_DESC luminance_desc = CD3D11_SHADER_RESOURCE_VIEW_DESC( - renderTexture.Get(), - D3D11_SRV_DIMENSION_TEXTURE2D, - (renderTextureDesc.Format == DXGI_FORMAT_P010) ? DXGI_FORMAT_R16_UNORM : DXGI_FORMAT_R8_UNORM - ); - DX::ThrowIfFailed(d3ddevice->CreateShaderResourceView( - renderTexture.Get(), &luminance_desc, m_luminance_shader_resource_view.GetAddressOf()), "Luminance SRV Creation" - ); - - D3D11_SHADER_RESOURCE_VIEW_DESC chrominance_desc = CD3D11_SHADER_RESOURCE_VIEW_DESC( - renderTexture.Get(), - D3D11_SRV_DIMENSION_TEXTURE2D, - (renderTextureDesc.Format == DXGI_FORMAT_P010) ? DXGI_FORMAT_R16G16_UNORM : DXGI_FORMAT_R8G8_UNORM - ); - DX::ThrowIfFailed(d3ddevice->CreateShaderResourceView( - renderTexture.Get(), &chrominance_desc, m_chrominance_shader_resource_view.GetAddressOf()), "Chrominance SRV Creation" - ); - d3dcontext->PSSetShaderResources(0, 1, m_luminance_shader_resource_view.GetAddressOf()); - d3dcontext->PSSetShaderResources(1, 1, m_chrominance_shader_resource_view.GetAddressOf()); - - this->bindColorConversion(frame); - - d3dcontext->DrawIndexed(6, 0, 0); + ctx->VSSetShader(m_vertexShader.Get(), nullptr, 0); + + bindColorConversion(frame); + + ctx->DrawIndexed(6, 0, 0); if (frame->color_trc != m_LastColorTrc) { DXGI_COLOR_SPACE_TYPE colorspace = {}; - if (frame->color_trc == AVCOL_TRC_SMPTE2084) { - // Switch to Rec 2020 PQ (SMPTE ST 2084) colorspace for HDR10 rendering + if (frame->color_trc == AVCOL_TRC_SMPTE2084) { + // Switch to Rec 2020 PQ (SMPTE ST 2084) colorspace for HDR10 rendering colorspace = DXGI_COLOR_SPACE_RGB_FULL_G2084_NONE_P2020; - } - else { - // Restore default sRGB colorspace + } else { + // Restore default sRGB colorspace colorspace = DXGI_COLOR_SPACE_RGB_FULL_G22_NONE_P709; - } + } UINT colorSpaceSupport = 0; - if (colorspace - && SUCCEEDED(m_deviceResources->GetSwapChain()->CheckColorSpaceSupport(colorspace, &colorSpaceSupport)) - && (colorSpaceSupport & DXGI_SWAP_CHAIN_COLOR_SPACE_SUPPORT_FLAG_PRESENT)) - { + if (colorspace && SUCCEEDED(m_deviceResources->GetSwapChain()->CheckColorSpaceSupport(colorspace, &colorSpaceSupport)) && (colorSpaceSupport & DXGI_SWAP_CHAIN_COLOR_SPACE_SUPPORT_FLAG_PRESENT)) { DX::ThrowIfFailed(m_deviceResources->GetSwapChain()->SetColorSpace1(colorspace)); Utils::Logf("Colorspace changed to %s\n", - colorspace == DXGI_COLOR_SPACE_RGB_FULL_G2084_NONE_P2020 - ? "DXGI_COLOR_SPACE_RGB_FULL_G2084_NONE_P2020" - : "DXGI_COLOR_SPACE_RGB_FULL_G22_NONE_P709"); + colorspace == DXGI_COLOR_SPACE_RGB_FULL_G2084_NONE_P2020 + ? "DXGI_COLOR_SPACE_RGB_FULL_G2084_NONE_P2020" + : "DXGI_COLOR_SPACE_RGB_FULL_G22_NONE_P709"); } - m_LastColorTrc = frame->color_trc; - } + m_LastColorTrc = frame->color_trc; + } - //UNLOCK_D3D(); + // UNLOCK_D3D(); return true; } - - void VideoRenderer::CreateDeviceDependentResources() { Utils::Log("Started with creation of DXView\n"); @@ -284,8 +284,8 @@ void VideoRenderer::CreateDeviceDependentResources() auto createCubeTask = (createVSTask && createPSTaskGen && createPSTaskBT601 && createPSTaskBT2020).then([this]() { Windows::Graphics::Display::Core::HdmiDisplayInformation^ hdi = Windows::Graphics::Display::Core::HdmiDisplayInformation::GetForCurrentView(); auto w = CoreWindow::GetForCurrentThread(); - int m_DisplayWidth = w->Bounds.Width; - int m_DisplayHeight = w->Bounds.Height; + int m_DisplayWidth = (int)w->Bounds.Width; + int m_DisplayHeight = (int)w->Bounds.Height; // HDR Setup if (hdi) { m_lastDisplayMode = hdi->GetCurrentDisplayMode(); @@ -348,24 +348,6 @@ void VideoRenderer::CreateDeviceDependentResources() DX::ThrowIfFailed(m_deviceResources->GetD3DDevice()->CreateBuffer(&indexBufferDesc, &indexBufferData, &m_indexBuffer), "Index Buffer creation"); }); - D3D11_RASTERIZER_DESC rasterizerState; - ZeroMemory(&rasterizerState, sizeof(D3D11_RASTERIZER_DESC)); - - rasterizerState.AntialiasedLineEnable = false; - rasterizerState.CullMode = D3D11_CULL_NONE; // D3D11_CULL_FRONT or D3D11_CULL_NONE D3D11_CULL_BACK - rasterizerState.FillMode = D3D11_FILL_SOLID; // D3D11_FILL_SOLID D3D11_FILL_WIREFRAME - rasterizerState.DepthBias = 0; - rasterizerState.DepthBiasClamp = 0.0f; - rasterizerState.DepthClipEnable = true; - rasterizerState.FrontCounterClockwise = false; - rasterizerState.MultisampleEnable = false; - rasterizerState.ScissorEnable = false; - rasterizerState.SlopeScaledDepthBias = 0.0f; - //rasterizerState.FillMode = D3D11_FILL_WIREFRAME; - ID3D11RasterizerState* m_pRasterState; - DX::ThrowIfFailed(m_deviceResources->GetD3DDevice()->CreateRasterizerState(&rasterizerState, &m_pRasterState), "Rasterizer Creation"); - m_deviceResources->GetD3DDeviceContext()->RSSetState(m_pRasterState); - D3D11_SAMPLER_DESC samplerDesc; samplerDesc.Filter = D3D11_FILTER_MIN_MAG_LINEAR_MIP_POINT; samplerDesc.AddressU = D3D11_TEXTURE_ADDRESS_CLAMP; @@ -419,7 +401,7 @@ void VideoRenderer::CreateDeviceDependentResources() dialog->ShowAsync(); return; } - m_loadingComplete = true; + m_loadingComplete.store(true, std::memory_order_release); Utils::Log("Loading Complete!\n"); }); } @@ -427,7 +409,7 @@ void VideoRenderer::CreateDeviceDependentResources() void VideoRenderer::ReleaseDeviceDependentResources() { Windows::Graphics::Display::Core::HdmiDisplayInformation^ hdi = Windows::Graphics::Display::Core::HdmiDisplayInformation::GetForCurrentView(); - m_loadingComplete = false; + m_loadingComplete.store(false, std::memory_order_release); m_vertexShader.Reset(); m_inputLayout.Reset(); m_pixelShaderGeneric.Reset(); @@ -440,8 +422,8 @@ void VideoRenderer::ReleaseDeviceDependentResources() void VideoRenderer::scaleSourceToDestinationSurface(IRECT* src, IRECT* dst) { - int dstH = ceil((float)dst->w * src->h / src->w); - int dstW = ceil((float)dst->h * src->w / src->h); + int dstH = (int)ceil((float)dst->w * src->h / src->w); + int dstW = (int)ceil((float)dst->h * src->w / src->h); if (dstH > dst->h) { dst->x += (dst->w - dstW) / 2; @@ -495,9 +477,7 @@ void VideoRenderer::bindColorConversion(AVFrame* frame) return; } - char msg[4096]; - sprintf_s(msg, "Falling back to generic video pixel shader for %d (%s range)\n", colorspace, fullRange ? "full" : "limited"); - Utils::Log(msg); + Utils::Logf("Falling back to generic video pixel shader for %d (%s range)\n", colorspace, fullRange ? "full" : "limited"); D3D11_BUFFER_DESC constDesc = {}; constDesc.ByteWidth = sizeof(CSC_CONST_BUF); diff --git a/Streaming/VideoRenderer.h b/Streaming/VideoRenderer.h index e6c9b1a7..a19856e9 100644 --- a/Streaming/VideoRenderer.h +++ b/Streaming/VideoRenderer.h @@ -30,6 +30,7 @@ namespace moonlight_xbox_dx void Update(DX::StepTimer const& timer); void scaleSourceToDestinationSurface(IRECT* src, IRECT* dst); void screenSpaceToNormalizedDeviceCoords(IRECT* src, FRECT* dst, int viewportWidth, int viewportHeight); + void EnsureYuvTargets(ID3D11Device* dev, DXGI_FORMAT fmt, UINT w, UINT h); bool Render(AVFrame* frame); void bindColorConversion(AVFrame* frame); void SetHDR(bool enabled); @@ -53,13 +54,19 @@ namespace moonlight_xbox_dx Windows::Graphics::Display::Core::HdmiDisplayMode^ m_lastDisplayMode; Windows::Graphics::Display::Core::HdmiDisplayMode^ m_currentDisplayMode; + Microsoft::WRL::ComPtr m_yuvTexture; + Microsoft::WRL::ComPtr m_srvY; + Microsoft::WRL::ComPtr m_srvUV; + DXGI_FORMAT m_yuvFormat = DXGI_FORMAT_UNKNOWN; + UINT m_yuvWidth = 0, m_yuvHeight = 0; + // System resources for cube geometry. ModelViewProjectionConstantBuffer m_constantBufferData; // Variables used with the rendering loop. AVColorTransferCharacteristic m_LastColorTrc; DXGI_HDR_METADATA_HDR10 m_lastHdr10; - bool m_loadingComplete; + std::atomic m_loadingComplete; bool m_LastFullRange; int m_LastColorSpace; MoonlightClient *client; diff --git a/Streaming/moonlight_xbox_dxMain.cpp b/Streaming/moonlight_xbox_dxMain.cpp index ce20a4ff..df56b7f4 100644 --- a/Streaming/moonlight_xbox_dxMain.cpp +++ b/Streaming/moonlight_xbox_dxMain.cpp @@ -112,24 +112,22 @@ void moonlight_xbox_dxMain::StartRenderLoop() Utils::Logf("Failed to set render thread priority: %d\n", GetLastError()); } + int64_t lastPresentTime = 0; + // Calculate the updated frame and render once per vertical blanking interval. while (action->Status == AsyncStatus::Started) { critical_section::scoped_lock lock(m_criticalSection); - LARGE_INTEGER beforeWait, afterWait, beforePresent; Update(); if (Render()) { - QueryPerformanceCounter(&beforePresent); + m_deviceResources->Present(); - static int64_t lastPresentTime = 0; + int64_t afterPresent = QpcNow(); if (lastPresentTime > 0) { - double frametime = QpcToMs(beforePresent.QuadPart - lastPresentTime); - ImGuiPlots::instance().observeFloat(PLOT_FRAMETIME, (float)frametime); + ImGuiPlots::instance().observeFloat(PLOT_FRAMETIME, (float)QpcToMs(afterPresent - lastPresentTime)); } - lastPresentTime = beforePresent.QuadPart; - - m_deviceResources->Present(); + lastPresentTime = afterPresent; } } }); From 496d5ab17d6ac00abbc665af97dff3ad15a63b58 Mon Sep 17 00:00:00 2001 From: Andy Grundman Date: Thu, 2 Oct 2025 07:35:15 -0400 Subject: [PATCH 15/22] Much more accurate timing for Present() --- Common/DeviceResources.cpp | 2 +- Streaming/FFmpegDecoder.cpp | 10 ++++- Streaming/FFmpegDecoder.h | 6 ++- Streaming/Pacer.cpp | 70 ++++++++++++++++++++--------- Streaming/Pacer.h | 3 ++ Streaming/moonlight_xbox_dxMain.cpp | 2 + 6 files changed, 67 insertions(+), 26 deletions(-) diff --git a/Common/DeviceResources.cpp b/Common/DeviceResources.cpp index bc6e274b..b5f980cd 100644 --- a/Common/DeviceResources.cpp +++ b/Common/DeviceResources.cpp @@ -318,7 +318,7 @@ void DX::DeviceResources::CreateWindowSizeDependentResources() swapChain.As(&m_swapChain) ); - m_swapChain->SetMaximumFrameLatency(2); + m_swapChain->SetMaximumFrameLatency(1); m_frameLatencyWaitable = m_swapChain->GetFrameLatencyWaitableObject(); // Associate swap chain with SwapChainPanel diff --git a/Streaming/FFmpegDecoder.cpp b/Streaming/FFmpegDecoder.cpp index e75bfa6c..4259b8bc 100644 --- a/Streaming/FFmpegDecoder.cpp +++ b/Streaming/FFmpegDecoder.cpp @@ -360,9 +360,15 @@ void FFMpegDecoder::WaitForFrame() { } } -bool FFMpegDecoder::RenderFrameOnMainThread(std::shared_ptr& sceneRenderer) { +bool FFMpegDecoder::RenderFrameOnMainThread(std::shared_ptr &sceneRenderer) { if (m_Pacer) { - return m_Pacer->renderOnMainThread(sceneRenderer); + return m_Pacer->renderOnMainThread(sceneRenderer); } return false; } + +void FFMpegDecoder::WaitUntilPresentTarget() { + if (m_Pacer) { + m_Pacer->waitUntilPresentTarget(); + } +} diff --git a/Streaming/FFmpegDecoder.h b/Streaming/FFmpegDecoder.h index f4db08eb..3d70ec06 100644 --- a/Streaming/FFmpegDecoder.h +++ b/Streaming/FFmpegDecoder.h @@ -17,8 +17,9 @@ extern "C" { #define MAX_BUFFER 1024 * 1024 typedef struct MLFrameData { - uint32_t presentationTimeMs; - int64_t decodeEndQpc; + uint32_t presentationTimeMs; // host's pts + int64_t decodeEndQpc; // when we finished decoding + int64_t presentTargetQpc; // timestamp when frame should be presented } MLFrameData; namespace moonlight_xbox_dx { @@ -36,6 +37,7 @@ class FFMpegDecoder { int ModifyFrameDropTarget(bool increase); void WaitForFrame(); bool RenderFrameOnMainThread(std::shared_ptr &sceneRenderer); + void WaitUntilPresentTarget(); static FFMpegDecoder *getInstance(); static DECODER_RENDERER_CALLBACKS getDecoder(); std::recursive_mutex mutex; diff --git a/Streaming/Pacer.cpp b/Streaming/Pacer.cpp index fafd6e68..ed9d3c1f 100644 --- a/Streaming/Pacer.cpp +++ b/Streaming/Pacer.cpp @@ -235,8 +235,8 @@ void Pacer::vsyncEmulator() { period.initFromHz(hzNum, hzDen, baseQpc); if (curDeadlineQpc > 0) { - Utils::Logf("resync to hardware vsync, frame: %d, drift: %lld ticks (%.3f ms), phase: %d\n", - frames, period.nextDeadlineQpc - curDeadlineQpc, QpcToMs(period.nextDeadlineQpc - curDeadlineQpc), phase); + FQLog("resync to hardware vsync, frame: %d, drift: %lld ticks (%.3f ms), phase: %d\n", + frames, period.nextDeadlineQpc - curDeadlineQpc, QpcToMs(period.nextDeadlineQpc - curDeadlineQpc), phase); } curDeadlineQpc = baseQpc; @@ -292,7 +292,6 @@ void Pacer::vsyncHardware() { } lastT0 = now; } - } void Pacer::backPacer() { @@ -405,29 +404,41 @@ void Pacer::handleVsync(int64_t nextDeadlineQpc) { // Place the first frame on the render queue AVFrame *frame = std::move(m_PacingQueue.front()); m_PacingQueue.pop_front(); - lock.unlock(); - // try to time this as close to deadline as possible - remainingQpc = nextDeadlineQpc - QpcNow(); - if (remainingQpc > 150) { - SleepUntilQpc(nextDeadlineQpc); + // Pass along the target timestamp with the frame metadata + if (frame->opaque_ref) { + auto *data = reinterpret_cast(frame->opaque_ref->data); + data->presentTargetQpc = nextDeadlineQpc; } + lock.unlock(); + + // Render frame immediately, it will wait to be presented at presentTargetQpc enqueueFrameForRenderingAndUnlock(frame); } +void Pacer::enqueueFrameForRenderingAndUnlock(AVFrame *frame) { + { + std::scoped_lock lock(m_FrameQueueLock); + dropFrameForEnqueue(m_RenderQueue, PLOT_DROPPED_PACER_FRONT); + m_RenderQueue.push_back(frame); + } + + // notify render loop of new frame + m_RenderQueueNotEmpty.notify_one(); +} + // Main thread void Pacer::waitForFrame() { // Wait for the renderer to be ready for the next frame - LARGE_INTEGER t0, t1; - QueryPerformanceCounter(&t0); + int64_t t0 = QpcNow(); HANDLE flw = m_DeviceResources->GetFrameLatencyWaitable(); WaitForSingleObjectEx(flw, 1000, true); - QueryPerformanceCounter(&t1); - FQLog("waitForFrame part 1 waited %.3f ms\n", QpcToMs(t1.QuadPart - t0.QuadPart)); + int64_t t1 = QpcNow(); + FQLog("waitForFrame(): FrameLatencyWaitable waited %.3f ms\n", QpcToMs(t1 - t0)); std::unique_lock lock(m_FrameQueueLock); m_RenderQueueNotEmpty.wait(lock, [this] { @@ -435,7 +446,7 @@ void Pacer::waitForFrame() { }); int64_t t2 = QpcNow(); - FQLog("waitForFrame part 2 waited %.3f ms\n", QpcToMs(t2 - t1.QuadPart)); + FQLog("waitForFrame(): m_RenderQueueNotEmpty waited %.3f ms\n", QpcToMs(t2 - t1)); } bool Pacer::renderOnMainThread(std::shared_ptr &sceneRenderer) { @@ -453,20 +464,37 @@ bool Pacer::renderOnMainThread(std::shared_ptr &sceneRenderer) { return false; // no frame, don't Present() } + // Extract the target present time for this frame + if (frame->opaque_ref) { + auto *data = reinterpret_cast(frame->opaque_ref->data); + m_PresentTargetQpc = data ? data->presentTargetQpc : 0; + } else { + m_PresentTargetQpc = 0; + } + frontPacer(sceneRenderer, frame); return true; // ok to Present() } -void Pacer::enqueueFrameForRenderingAndUnlock(AVFrame *frame) { - { - std::scoped_lock lock(m_FrameQueueLock); - dropFrameForEnqueue(m_RenderQueue, PLOT_DROPPED_PACER_FRONT); - m_RenderQueue.push_back(frame); +void Pacer::waitUntilPresentTarget() { + const int64_t target = m_PresentTargetQpc; + if (target <= 0) { + return; } - // notify render loop of new frame - FQLog("enqueueFrame notifying m_RenderQueueNotEmpty\n"); - m_RenderQueueNotEmpty.notify_one(); + const int64_t now = QpcNow(); + if (target <= now) { + FQLog("waitUntilPresentTarget(): target was %.3f ms too late\n", QpcToMs(now - target)); + return; + } + + FQLog("waitUntilPresentTarget(): waiting for %.3f ms\n", QpcToMs(target - now)); + + SleepUntilQpc(target); + + // Measure how well we timed things + // const double skewMs = QpcToMs(QpcNow() - target); + // ImGuiPlots::instance().observeFloat(PLOT_PRESENT_ACCURACY, (float)skewMs); } void Pacer::frontPacer(std::shared_ptr &sceneRenderer, AVFrame *frame) { diff --git a/Streaming/Pacer.h b/Streaming/Pacer.h index 60223a1c..530cc81c 100644 --- a/Streaming/Pacer.h +++ b/Streaming/Pacer.h @@ -24,6 +24,8 @@ class Pacer { bool renderOnMainThread(std::shared_ptr &sceneRenderer); + void waitUntilPresentTarget(); + int getFrameDropTarget(); int modifyFrameDropTarget(bool increase); @@ -63,6 +65,7 @@ class Pacer { std::mutex m_FrameQueueLock; std::condition_variable m_RenderQueueNotEmpty; std::condition_variable m_PacingQueueNotEmpty; + int64_t m_PresentTargetQpc; std::atomic m_Stopping{false}; int m_StreamFps; double m_RefreshRate; diff --git a/Streaming/moonlight_xbox_dxMain.cpp b/Streaming/moonlight_xbox_dxMain.cpp index df56b7f4..22704c46 100644 --- a/Streaming/moonlight_xbox_dxMain.cpp +++ b/Streaming/moonlight_xbox_dxMain.cpp @@ -121,6 +121,8 @@ void moonlight_xbox_dxMain::StartRenderLoop() Update(); if (Render()) { + FFMpegDecoder::instance().WaitUntilPresentTarget(); + m_deviceResources->Present(); int64_t afterPresent = QpcNow(); From faa56bcc42d0380aebd4c734a13b544be3c4c56d Mon Sep 17 00:00:00 2001 From: Andy Grundman Date: Sat, 4 Oct 2025 17:38:26 -0400 Subject: [PATCH 16/22] Remove option to select vsync, instead force it off for 120hz, on for 30/60. Remove commented out D3D locking code. --- Common/DeviceResources.cpp | 4 ---- Pages/HostSettingsPage.xaml | 5 +++-- Pages/StreamPage.xaml.cpp | 8 +++++++- Streaming/FFmpegDecoder.cpp | 11 ----------- Streaming/FFmpegDecoder.h | 18 ------------------ Streaming/VideoRenderer.cpp | 4 ---- Streaming/moonlight_xbox_dxMain.cpp | 9 --------- 7 files changed, 10 insertions(+), 49 deletions(-) diff --git a/Common/DeviceResources.cpp b/Common/DeviceResources.cpp index b5f980cd..17d7c761 100644 --- a/Common/DeviceResources.cpp +++ b/Common/DeviceResources.cpp @@ -566,8 +566,6 @@ void DX::DeviceResources::Present() HRESULT hr = E_FAIL; DXGI_PRESENT_PARAMETERS parameters = { 0 }; - //LOCK_D3D("Present"); - if (m_enableVsync) { // Composition swapchain hr = m_swapChain->Present1(0, 0, ¶meters); @@ -586,8 +584,6 @@ void DX::DeviceResources::Present() // overwritten. If dirty or scroll rects are used, this call should be modified. m_d3dContext->DiscardView1(m_d3dRenderTargetView.Get(), nullptr, 0); - //UNLOCK_D3D(); - // If the device was removed either by a disconnection or a driver upgrade, we // must recreate all device resources. if (hr == DXGI_ERROR_DEVICE_REMOVED || hr == DXGI_ERROR_DEVICE_RESET) diff --git a/Pages/HostSettingsPage.xaml b/Pages/HostSettingsPage.xaml index 5b9e69f7..1a096f55 100644 --- a/Pages/HostSettingsPage.xaml +++ b/Pages/HostSettingsPage.xaml @@ -70,8 +70,9 @@ Optimize host resolution: - Enable V-Sync: - + + Show performance stats: (e->Parameter); SetStreamConfig(configuration); - m_deviceResources->SetVsync(configuration->enableVsync); + + if (configuration->FPS > 60) { + m_deviceResources->SetVsync(false); + } else { + m_deviceResources->SetVsync(true); + } + if (configuration == nullptr)return; } diff --git a/Streaming/FFmpegDecoder.cpp b/Streaming/FFmpegDecoder.cpp index 4259b8bc..2cee208b 100644 --- a/Streaming/FFmpegDecoder.cpp +++ b/Streaming/FFmpegDecoder.cpp @@ -59,14 +59,6 @@ namespace moonlight_xbox_dx { m_Pacer(nullptr) { } - void lock_context(void*) { - LOCK_D3D("ffmpeg"); - } - - void unlock_context(void*) { - UNLOCK_D3D(); - } - void ffmpeg_log_callback(void *ptr, int level, const char *fmt, va_list vl) { char lineBuffer[1024]; static int printPrefix = 1; @@ -135,9 +127,6 @@ namespace moonlight_xbox_dx { d3d11va_device_ctx = reinterpret_cast(device_ctx->hwctx); d3d11va_device_ctx->device = this->resources->GetD3DDevice(); d3d11va_device_ctx->device_context = this->resources->GetD3DDeviceContext(); - d3d11va_device_ctx->lock = lock_context; - d3d11va_device_ctx->unlock = unlock_context; - d3d11va_device_ctx->lock_ctx = nullptr; int err2; if ((err2 = av_hwdevice_ctx_init(hw_device_ctx)) < 0) { char msg[2048]; diff --git a/Streaming/FFmpegDecoder.h b/Streaming/FFmpegDecoder.h index 3d70ec06..613dd43f 100644 --- a/Streaming/FFmpegDecoder.h +++ b/Streaming/FFmpegDecoder.h @@ -60,21 +60,3 @@ class FFMpegDecoder { std::unique_ptr m_Pacer; }; } // namespace moonlight_xbox_dx - -inline void LOCK_D3D(const char *msg) { -#ifdef FRAME_QUEUE_VERBOSE - LARGE_INTEGER t0, t1; - QueryPerformanceCounter(&t0); -#endif - - moonlight_xbox_dx::FFMpegDecoder::instance().mutex.lock(); - -#ifdef FRAME_QUEUE_VERBOSE - QueryPerformanceCounter(&t1); - //FQLog("%s acquired lock_context in %.3f ms\n", msg, QpcToMs(t1.QuadPart - t0.QuadPart)); -#endif -} - -inline void UNLOCK_D3D() { - moonlight_xbox_dx::FFMpegDecoder::instance().mutex.unlock(); -} diff --git a/Streaming/VideoRenderer.cpp b/Streaming/VideoRenderer.cpp index 4d9ae2d0..bc777bd5 100644 --- a/Streaming/VideoRenderer.cpp +++ b/Streaming/VideoRenderer.cpp @@ -149,8 +149,6 @@ bool VideoRenderer::Render(AVFrame *frame) { return true; } - // LOCK_D3D("VideoRenderer->Render"); - auto *ctx = m_deviceResources->GetD3DDeviceContext(); auto *dev = m_deviceResources->GetD3DDevice(); @@ -209,8 +207,6 @@ bool VideoRenderer::Render(AVFrame *frame) { m_LastColorTrc = frame->color_trc; } - // UNLOCK_D3D(); - return true; } diff --git a/Streaming/moonlight_xbox_dxMain.cpp b/Streaming/moonlight_xbox_dxMain.cpp index 22704c46..454e2b4e 100644 --- a/Streaming/moonlight_xbox_dxMain.cpp +++ b/Streaming/moonlight_xbox_dxMain.cpp @@ -383,36 +383,27 @@ bool moonlight_xbox_dxMain::Render() return false; } - //LOCK_D3D("Clear"); Clear(); - //UNLOCK_D3D(); // Start the Dear ImGui frame, limited to only running at 60fps bool showImGui = m_deviceResources->GetShowImGui(); if (showImGui) { - //LOCK_D3D("ImGui 1"); ImGui_ImplDX11_NewFrame(); - //UNLOCK_D3D(); ImGui_ImplUwp_NewFrame(m_deviceResources->GetPixelWidth(), m_deviceResources->GetPixelHeight()); ImGui::NewFrame(); } // Render the scene objects. - //LOCK_D3D("Render"); - FFMpegDecoder::instance().WaitForFrame(); bool shouldPresent = FFMpegDecoder::instance().RenderFrameOnMainThread(m_sceneRenderer); m_LogRenderer->Render(); m_statsTextRenderer->Render(showImGui); - //UNLOCK_D3D(); if (showImGui) { RenderImGui(); ImGui::Render(); - //LOCK_D3D("ImGui 2"); ImGui_ImplDX11_RenderDrawData(ImGui::GetDrawData()); - //UNLOCK_D3D(); } return shouldPresent; From 6a75967ab466bc302e90cd49301c18dcb191c180 Mon Sep 17 00:00:00 2001 From: Andy Grundman Date: Mon, 13 Oct 2025 12:05:37 -0400 Subject: [PATCH 17/22] Host Settings: bitrate calc from res/fps, add message about HDR when not at 4K, clean out vsync setting, fixup clientRefreshRateX100 for non-NTSC. --- Pages/AppPage.xaml.cpp | 1 - Pages/HostSettingsPage.xaml | 12 +++- Pages/HostSettingsPage.xaml.cpp | 100 +++++++++++++++++++++++++++++--- Pages/HostSettingsPage.xaml.h | 4 +- Pages/StreamPage.xaml | 12 ---- Pages/StreamPage.xaml.cpp | 16 ----- Pages/StreamPage.xaml.h | 2 - State/ApplicationState.cpp | 2 - State/MoonlightClient.cpp | 15 +++-- State/MoonlightHost.h | 11 ---- 10 files changed, 116 insertions(+), 59 deletions(-) diff --git a/Pages/AppPage.xaml.cpp b/Pages/AppPage.xaml.cpp index 9d4ef7ab..a10e2063 100644 --- a/Pages/AppPage.xaml.cpp +++ b/Pages/AppPage.xaml.cpp @@ -77,7 +77,6 @@ void AppPage::Connect(int appId) { if (config->enableHDR) { host->VideoCodec = "HEVC (H.265)"; } - config->enableVsync = host->EnableVsync; bool result = this->Frame->Navigate(Windows::UI::Xaml::Interop::TypeName(StreamPage::typeid), config); if (!result) { printf("C"); diff --git a/Pages/HostSettingsPage.xaml b/Pages/HostSettingsPage.xaml index 1a096f55..a71f90e8 100644 --- a/Pages/HostSettingsPage.xaml +++ b/Pages/HostSettingsPage.xaml @@ -53,7 +53,7 @@ FPS - + Bitrate Audio Configuration @@ -65,8 +65,16 @@ Video Codecs + Enable HDR: - + + + HDR requires Xbox system resolution set to 4K. + + Optimize host resolution: diff --git a/Pages/HostSettingsPage.xaml.cpp b/Pages/HostSettingsPage.xaml.cpp index 9ace6e45..69e70ba5 100644 --- a/Pages/HostSettingsPage.xaml.cpp +++ b/Pages/HostSettingsPage.xaml.cpp @@ -15,6 +15,7 @@ using namespace moonlight_xbox_dx; using namespace Platform; using namespace Windows::Foundation; using namespace Windows::Foundation::Collections; +using namespace Windows::Graphics::Display::Core; using namespace Windows::UI::Xaml; using namespace Windows::UI::Xaml::Controls; using namespace Windows::UI::Xaml::Controls::Primitives; @@ -76,12 +77,27 @@ void HostSettingsPage::OnNavigatedTo(Windows::UI::Xaml::Navigation::NavigationEv } } AutoStartSelector->SelectedIndex = CurrentAppIndex; - //Old Xbox One can only use H264, remove from settings everything else - if ((info.vendorId == GAMING_DEVICE_VENDOR_ID_MICROSOFT && info.deviceId == GAMING_DEVICE_DEVICE_ID_XBOX_ONE)) { - CodecComboBox->IsEnabled = false; - CodecComboBox->SelectedIndex = 0; - EnableHDRCheckbox->IsChecked = false; - EnableHDRCheckbox->IsEnabled = false; + + if (info.vendorId == GAMING_DEVICE_VENDOR_ID_MICROSOFT) { + // Old Xbox One can only use H264, remove from settings everything else + if (info.deviceId == GAMING_DEVICE_DEVICE_ID_XBOX_ONE) { + CodecComboBox->IsEnabled = false; + CodecComboBox->SelectedIndex = 0; + } + + // Disable HDR if console is not set to 4K + auto mode = HdmiDisplayInformation::GetForCurrentView()->GetCurrentDisplayMode(); + auto height = mode->ResolutionHeightInRawPixels; + if (height < 2160) { + EnableHDRCheckbox->IsEnabled = false; + EnableHDRCheckbox->IsChecked = false; + EnableHDRCheckbox->Visibility = Windows::UI::Xaml::Visibility::Collapsed; + HDR4KNote->Visibility = Windows::UI::Xaml::Visibility::Visible; + } else { + EnableHDRCheckbox->IsEnabled = true; + EnableHDRCheckbox->Visibility = Windows::UI::Xaml::Visibility::Visible; + HDR4KNote->Visibility = Windows::UI::Xaml::Visibility::Collapsed; + } } } @@ -104,7 +120,26 @@ void HostSettingsPage::OnBackRequested(Platform::Object^ e, Windows::UI::Core::B void HostSettingsPage::ResolutionSelector_SelectionChanged(Platform::Object^ sender, Windows::UI::Xaml::Controls::SelectionChangedEventArgs^ e) { - host->Resolution = AvailableResolutions->GetAt(this->ResolutionSelector->SelectedIndex); + auto selectedResolution = AvailableResolutions->GetAt(this->ResolutionSelector->SelectedIndex); + + // Default to a new bitrate if a new resolution was chosen + if (selectedResolution->Width != host->Resolution->Width) { + host->Bitrate = getDefaultBitrate(selectedResolution->Width, selectedResolution->Height, host->FPS); + } + + host->Resolution = selectedResolution; +} + +void HostSettingsPage::FPSSelector_SelectionChanged(Platform::Object^ sender, Windows::UI::Xaml::Controls::SelectionChangedEventArgs^ e) +{ + int selectedFPS = (int)e->AddedItems->GetAt(0); + + // Default to a new bitrate if a new FPS was chosen + if (selectedFPS != host->FPS) { + host->Bitrate = getDefaultBitrate(host->Resolution->Width, host->Resolution->Height, selectedFPS); + } + + host->FPS = selectedFPS; } void HostSettingsPage::AutoStartSelector_SelectionChanged(Platform::Object^ sender, Windows::UI::Xaml::Controls::SelectionChangedEventArgs^ e) @@ -142,4 +177,53 @@ void HostSettingsPage::OnUnloaded(Platform::Object^ sender, Windows::UI::Xaml::R { auto navigation = Windows::UI::Core::SystemNavigationManager::GetForCurrentView(); navigation->BackRequested -= m_back_cookie; -} \ No newline at end of file +} + +int HostSettingsPage::getDefaultBitrate(int width, int height, int fps) +{ + // Don't scale bitrate linearly beyond 60 FPS. It's definitely not a linear + // bitrate increase for frame rate once we get to values that high. + float frameRateFactor = (fps <= 60 ? fps : (std::sqrtf(fps / 60.f) * 60.f)) / 30.f; + + // TODO: Collect some empirical data to see if these defaults make sense. + // We're just using the values that the Shield used, as we have for years. + static const struct resTable { + int pixels; + int factor; + } resTable[] { + { 1280 * 720, 5 }, + { 1920 * 1080, 10 }, + { 2560 * 1440, 20 }, + { 3840 * 2160, 40 }, + { -1, -1 }, + }; + + // Calculate the resolution factor by linear interpolation of the resolution table + float resolutionFactor; + int pixels = width * height; + for (int i = 0;; i++) { + if (pixels == resTable[i].pixels) { + // We can bail immediately for exact matches + resolutionFactor = resTable[i].factor; + break; + } + else if (pixels < resTable[i].pixels) { + if (i == 0) { + // Never go below the lowest resolution entry + resolutionFactor = resTable[i].factor; + } + else { + // Interpolate between the entry greater than the chosen resolution (i) and the entry less than the chosen resolution (i-1) + resolutionFactor = ((float)(pixels - resTable[i-1].pixels) / (resTable[i].pixels - resTable[i-1].pixels)) * (resTable[i].factor - resTable[i-1].factor) + resTable[i-1].factor; + } + break; + } + else if (resTable[i].pixels == -1) { + // Never go above the highest resolution entry + resolutionFactor = resTable[i-1].factor; + break; + } + } + + return std::lround(resolutionFactor * frameRateFactor) * 1000; +} diff --git a/Pages/HostSettingsPage.xaml.h b/Pages/HostSettingsPage.xaml.h index ff6c9d40..a69c8bc2 100644 --- a/Pages/HostSettingsPage.xaml.h +++ b/Pages/HostSettingsPage.xaml.h @@ -96,11 +96,13 @@ namespace moonlight_xbox_dx private: void backButton_Click(Platform::Object^ sender, Windows::UI::Xaml::RoutedEventArgs^ e); void ResolutionSelector_SelectionChanged(Platform::Object^ sender, Windows::UI::Xaml::Controls::SelectionChangedEventArgs^ e); + void FPSSelector_SelectionChanged(Platform::Object^ sender, Windows::UI::Xaml::Controls::SelectionChangedEventArgs^ e); void BitrateInput_TextChanged(Platform::Object^ sender, Windows::UI::Xaml::Controls::TextChangedEventArgs^ e); void AutoStartSelector_SelectionChanged(Platform::Object^ sender, Windows::UI::Xaml::Controls::SelectionChangedEventArgs^ e); void GlobalSettingsOption_Click(Platform::Object^ sender, Windows::UI::Xaml::RoutedEventArgs^ e); void BitrateInput_KeyDown(Platform::Object^ sender, Windows::UI::Xaml::Input::KeyRoutedEventArgs^ e); void OnLoaded(Platform::Object^ sender, Windows::UI::Xaml::RoutedEventArgs^ e); void OnUnloaded(Platform::Object^ sender, Windows::UI::Xaml::RoutedEventArgs^ e); + int getDefaultBitrate(int width, int height, int fps); }; -} \ No newline at end of file +} diff --git a/Pages/StreamPage.xaml b/Pages/StreamPage.xaml index 3ef01576..bf97f037 100644 --- a/Pages/StreamPage.xaml +++ b/Pages/StreamPage.xaml @@ -45,18 +45,6 @@ - - - - - - - - - - - - diff --git a/Pages/StreamPage.xaml.cpp b/Pages/StreamPage.xaml.cpp index 951275a4..ec62d387 100644 --- a/Pages/StreamPage.xaml.cpp +++ b/Pages/StreamPage.xaml.cpp @@ -236,28 +236,12 @@ void StreamPage::toggleHDR_WinAltB_Click(Platform::Object^ sender, Windows::UI:: this->m_main->SendWinAltB(); } -void StreamPage::increaseFrameDropTarget_Click(Platform::Object^ sender, Windows::UI::Xaml::RoutedEventArgs^ e) -{ - int target = FFMpegDecoder::instance().ModifyFrameDropTarget(true); // true increases the value - this->frameDropTargetLabel->Text = Utils::StringPrintf("Frame queue size: %d", target); -} - -void StreamPage::decreaseFrameDropTarget_Click(Platform::Object^ sender, Windows::UI::Xaml::RoutedEventArgs^ e) -{ - int target = FFMpegDecoder::instance().ModifyFrameDropTarget(false); // false decreases the value - this->frameDropTargetLabel->Text = Utils::StringPrintf("Frame queue size: %d", target); -} - void StreamPage::OnLoaded(Platform::Object^ sender, Windows::UI::Xaml::RoutedEventArgs^ e) { auto navigation = Windows::UI::Core::SystemNavigationManager::GetForCurrentView(); m_back_cookie = navigation->BackRequested += ref new EventHandler(this, &StreamPage::OnBackRequested); - - int target = FFMpegDecoder::instance().GetFrameDropTarget(); - this->frameDropTargetLabel->Text = Utils::StringPrintf("Frame queue size: %d", target); } - void StreamPage::OnUnloaded(Platform::Object^ sender, Windows::UI::Xaml::RoutedEventArgs^ e) { auto navigation = Windows::UI::Core::SystemNavigationManager::GetForCurrentView(); diff --git a/Pages/StreamPage.xaml.h b/Pages/StreamPage.xaml.h index e6000c7b..75150602 100644 --- a/Pages/StreamPage.xaml.h +++ b/Pages/StreamPage.xaml.h @@ -97,8 +97,6 @@ namespace moonlight_xbox_dx void guideButtonShort_Click(Platform::Object^ sender, Windows::UI::Xaml::RoutedEventArgs^ e); void guideButtonLong_Click(Platform::Object^ sender, Windows::UI::Xaml::RoutedEventArgs^ e); void toggleHDR_WinAltB_Click(Platform::Object^ sender, Windows::UI::Xaml::RoutedEventArgs^ e); - void increaseFrameDropTarget_Click(Platform::Object^ sender, Windows::UI::Xaml::RoutedEventArgs^ e); - void decreaseFrameDropTarget_Click(Platform::Object^ sender, Windows::UI::Xaml::RoutedEventArgs^ e); void OnLoaded(Platform::Object^ sender, Windows::UI::Xaml::RoutedEventArgs^ e); void OnUnloaded(Platform::Object^ sender, Windows::UI::Xaml::RoutedEventArgs^ e); }; diff --git a/State/ApplicationState.cpp b/State/ApplicationState.cpp index a34d7019..3d36dbcd 100644 --- a/State/ApplicationState.cpp +++ b/State/ApplicationState.cpp @@ -47,7 +47,6 @@ Concurrency::task moonlight_xbox_dx::ApplicationState::Init() if (a.contains("enable_sops")) h->EnableSOPS = a["enable_sops"].get(); if (a.contains("enable_stats")) h->EnableStats = a["enable_stats"].get(); if (a.contains("enable_graphs")) h->EnableGraphs = a["enable_graphs"].get(); - if (a.contains("enable_vsync")) h->EnableVsync = a["enable_vsync"].get(); if (a.contains("serverAddress")) h->ServerAddress = Utils::StringFromStdString(a["serverAddress"].get()); if (a.contains("macaddress")) h->MacAddress = Utils::StringFromStdString(a["macaddress"].get()); else h->ComputerName = h->LastHostname; @@ -107,7 +106,6 @@ Concurrency::task moonlight_xbox_dx::ApplicationState::UpdateFile() hostJson["enable_sops"] = host->EnableSOPS; hostJson["enable_stats"] = host->EnableStats; hostJson["enable_graphs"] = host->EnableGraphs; - hostJson["enable_vsync"] = host->EnableVsync; hostJson["serverAddress"] = Utils::PlatformStringToStdString(host->ServerAddress); std::string macAddr = Utils::PlatformStringToStdString(host->MacAddress); diff --git a/State/MoonlightClient.cpp b/State/MoonlightClient.cpp index 1c10b59e..66f1e73b 100644 --- a/State/MoonlightClient.cpp +++ b/State/MoonlightClient.cpp @@ -221,20 +221,27 @@ int MoonlightClient::StartStreaming(std::shared_ptr res, St config.fps = sConfig->FPS; if (res->GetRefreshRate() > 0.0 && info.vendorId == GAMING_DEVICE_VENDOR_ID_MICROSOFT) { // Pass fractional refresh rate to host in case it's supported + double rr = res->GetRefreshRate(); switch (config.fps) { case 120: - config.clientRefreshRateX100 = (int)(res->GetRefreshRate() * 100.0); + config.clientRefreshRateX100 = (int)(rr * 100.0); break; case 60: - config.clientRefreshRateX100 = 5994; // enable 60fps stream in 120hz + if (rr >= 120.0) { + config.clientRefreshRateX100 = 6000; + } else if (rr >= 119.0) { + config.clientRefreshRateX100 = 5994; + } else { + config.clientRefreshRateX100 = (int)(rr * 100.0); + } break; default: config.clientRefreshRateX100 = sConfig->FPS * 100; break; } - Utils::Logf("Requesting stream with clientRefreshRateX100=%d for %d FPS within %.2f Hz\n", - config.clientRefreshRateX100, config.fps, res->GetRefreshRate()); + Utils::Logf("Requesting stream with clientRefreshRateX100=%d for %d FPS on %.2f Hz display\n", + config.clientRefreshRateX100, config.fps, rr); } config.colorRange = this->IsRGBFull() ? COLOR_RANGE_FULL : COLOR_RANGE_LIMITED; config.colorSpace = COLORSPACE_REC_601; diff --git a/State/MoonlightHost.h b/State/MoonlightHost.h index b617e416..56d6af42 100644 --- a/State/MoonlightHost.h +++ b/State/MoonlightHost.h @@ -29,7 +29,6 @@ namespace moonlight_xbox_dx { bool enableSOPS = false; bool enableStats = false; bool enableGraphs = true; - bool enableVsync = false; Windows::Foundation::Collections::IVector^ apps; public: //Thanks to https://phsucharee.wordpress.com/2013/06/19/data-binding-and-ccx-inotifypropertychanged/ @@ -254,15 +253,5 @@ namespace moonlight_xbox_dx { OnPropertyChanged("EnableGraphs"); } } - - // NOTE: vsync is currently forced ON until we can figure out how to get VRR working - property bool EnableVsync - { - bool get() { return this->enableVsync; } - void set(bool value) { - this->enableVsync = value; - OnPropertyChanged("EnableVsync"); - } - } }; } From 204ccc17180c9077e493643faa961909d98e68f5 Mon Sep 17 00:00:00 2001 From: Andy Grundman Date: Mon, 13 Oct 2025 12:10:48 -0400 Subject: [PATCH 18/22] LogRenderer: add warning about reduced performance when visible --- Streaming/LogRenderer.cpp | 15 ++++++++++++++- Streaming/LogRenderer.h | 1 + 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/Streaming/LogRenderer.cpp b/Streaming/LogRenderer.cpp index c8a754d3..62040e5b 100644 --- a/Streaming/LogRenderer.cpp +++ b/Streaming/LogRenderer.cpp @@ -11,11 +11,13 @@ using namespace Microsoft::WRL; LogRenderer::LogRenderer(const std::shared_ptr& deviceResources) : m_console(std::make_unique()), + m_warningConsole(std::make_unique()), m_deviceResources(deviceResources), m_mutex(), m_visible(false) { m_console->SetForegroundColor(Colors::Yellow); + m_warningConsole->SetForegroundColor(Colors::Red); //m_console->SetDebugOutput(true); CreateDeviceDependentResources(); @@ -30,6 +32,7 @@ void LogRenderer::Update(DX::StepTimer const& timer) static double lastUpdateSeconds = 0.0; if (m_visible && timer.GetTotalSeconds() - lastUpdateSeconds >= 1.0) { m_console->Clear(); + m_warningConsole->Clear(); Utils::logMutex.lock(); std::vector lines = Utils::GetLogLines(); @@ -38,6 +41,8 @@ void LogRenderer::Update(DX::StepTimer const& timer) } Utils::logMutex.unlock(); + m_warningConsole->Write(L"Warning: Viewing logs reduces performance"); + lastUpdateSeconds = timer.GetTotalSeconds(); } } @@ -48,6 +53,7 @@ void LogRenderer::Render() std::lock_guard lock(m_mutex); if (m_visible) { m_console->Render(); + m_warningConsole->Render(); } } @@ -61,22 +67,29 @@ void LogRenderer::CreateDeviceDependentResources() } m_console->RestoreDevice(m_deviceResources->GetD3DDeviceContext(), font); + m_warningConsole->RestoreDevice(m_deviceResources->GetD3DDeviceContext(), L"Assets\\Font\\ModeSeven-24.spritefont"); // use much faster font rendering m_console->SetFixedWidthFont(true); + m_warningConsole->SetFixedWidthFont(true); } void LogRenderer::CreateWindowSizeDependentResources() { // The size of our text area (left, top, right, bottom) - RECT size = {m_displayWidth * 0.5, 0, m_displayWidth - 20, m_displayHeight - 20}; + RECT size = {m_displayWidth * 0.5, 0, m_displayWidth - 20, m_displayHeight - 44}; m_console->SetWindow(size); + + // The size of our text area (left, top, right, bottom) + RECT warningSize = {size.left, m_displayHeight - 44, size.right, m_displayHeight - 20}; + m_warningConsole->SetWindow(warningSize); } void LogRenderer::ReleaseDeviceDependentResources() { m_console->ReleaseDevice(); + m_warningConsole->ReleaseDevice(); } void LogRenderer::ToggleVisible() { diff --git a/Streaming/LogRenderer.h b/Streaming/LogRenderer.h index 3f976095..838a3d9f 100644 --- a/Streaming/LogRenderer.h +++ b/Streaming/LogRenderer.h @@ -23,6 +23,7 @@ namespace moonlight_xbox_dx std::mutex m_mutex; std::shared_ptr m_deviceResources; std::unique_ptr m_console; + std::unique_ptr m_warningConsole; bool m_visible; uint32_t m_displayWidth; uint32_t m_displayHeight; From c0859928643555cf97ce8b3fb54f9a8ee6671d47 Mon Sep 17 00:00:00 2001 From: Andy Grundman Date: Wed, 8 Oct 2025 15:44:44 -0400 Subject: [PATCH 19/22] Pacer refactoring, added helper for timing Present to screen tear line. Misc minor fixes. --- .clang-format | 3 +- Common/DeviceResources.cpp | 17 +- Common/TextConsole.cpp | 4 + Plot/ImGuiPlots.cpp | 6 +- Plot/PlotDesc.h | 24 +- State/Stats.cpp | 12 +- State/Stats.h | 2 + Streaming/FFmpegDecoder.cpp | 44 +-- Streaming/FFmpegDecoder.h | 9 +- Streaming/Pacer.cpp | 453 +++++++++++++++++----------- Streaming/Pacer.h | 63 ++-- Streaming/PacerRational.h | 2 +- Streaming/PacerTearController.h | 100 ++++++ Streaming/StatsRenderer.cpp | 75 +++-- Streaming/moonlight_xbox_dxMain.cpp | 16 +- Streaming/moonlight_xbox_dxMain.h | 2 + Utils/FloatBuffer.cpp | 2 +- Utils/FloatBuffer.h | 2 +- pch.h | 51 ++-- 19 files changed, 544 insertions(+), 343 deletions(-) create mode 100644 Streaming/PacerTearController.h diff --git a/.clang-format b/.clang-format index 4782c9af..59937f48 100644 --- a/.clang-format +++ b/.clang-format @@ -12,7 +12,8 @@ AlignConsecutiveAssignments: false AlignConsecutiveDeclarations: false AlignTrailingComments: true ConstructorInitializerAllOnOneLineOrOnePerLine: true -BreakConstructorInitializers: AfterColon +BreakConstructorInitializers: BeforeColon +PackConstructorInitializers: Never # Don't move pch.h below other headers IncludeBlocks: Preserve diff --git a/Common/DeviceResources.cpp b/Common/DeviceResources.cpp index 17d7c761..f843caf1 100644 --- a/Common/DeviceResources.cpp +++ b/Common/DeviceResources.cpp @@ -75,7 +75,8 @@ DX::DeviceResources::DeviceResources() : m_deviceNotify(nullptr), m_stats(nullptr), m_imguiRunning(false), - m_showImGui(false) + m_showImGui(false), + m_frameLatencyWaitable() { m_refreshRate = GetUWPRefreshRate(); GetUWPPixelDimensions(&m_pixelWidth, &m_pixelHeight); @@ -230,8 +231,8 @@ void DX::DeviceResources::CreateWindowSizeDependentResources() UpdateRenderTargetSize(); //Get the correct screen resolution and adapt the swapchain to 16:9 aspect ratio - float normalizedWidth = uwp_get_width(); - float normalizedHeight = uwp_get_height(); + float normalizedWidth = (float)uwp_get_width(); + float normalizedHeight = (float)uwp_get_height(); m_d3dRenderTargetSize.Width = normalizedWidth; m_d3dRenderTargetSize.Height = normalizedHeight; @@ -384,13 +385,13 @@ void DX::DeviceResources::UpdateRenderTargetSize() { auto state = GetApplicationState(); m_effectiveDpi = m_dpi; - double compositionScaleMultiplier = 1; + float compositionScaleMultiplier = 1.0f; Windows::Graphics::Display::Core::HdmiDisplayInformation^ hdi = Windows::Graphics::Display::Core::HdmiDisplayInformation::GetForCurrentView(); if (hdi) { auto mode = hdi->GetCurrentDisplayMode(); - if (mode->ResolutionWidthInRawPixels > 2560)compositionScaleMultiplier = 2; - else if (mode->ResolutionWidthInRawPixels > 1920)compositionScaleMultiplier = 1.33333333; + if (mode->ResolutionWidthInRawPixels > 2560) compositionScaleMultiplier = 2.0f; + else if (mode->ResolutionWidthInRawPixels > 1920) compositionScaleMultiplier = 1.33333333f; } m_effectiveCompositionScaleX = m_compositionScaleX * compositionScaleMultiplier; @@ -674,8 +675,8 @@ void DX::DeviceResources::GetUWPPixelDimensions(uint32_t *width, uint32_t *heigh // Running in Windows const LONG32 resolution_scale = static_cast(DisplayInformation::GetForCurrentView()->ResolutionScale); auto surface_scale = static_cast(resolution_scale) / 100.0f; - *width = CoreWindow::GetForCurrentThread()->Bounds.Width * surface_scale; - *height = CoreWindow::GetForCurrentThread()->Bounds.Height * surface_scale; + *width = (uint32_t)CoreWindow::GetForCurrentThread()->Bounds.Width * surface_scale; + *height = (uint32_t)CoreWindow::GetForCurrentThread()->Bounds.Height * surface_scale; } } diff --git a/Common/TextConsole.cpp b/Common/TextConsole.cpp index 9899abb0..780d1c74 100644 --- a/Common/TextConsole.cpp +++ b/Common/TextConsole.cpp @@ -23,6 +23,7 @@ using Microsoft::WRL::ComPtr; using namespace DirectX; using namespace DX; +using namespace moonlight_xbox_dx; TextConsole::TextConsole() noexcept : m_layout{}, @@ -200,6 +201,9 @@ void TextConsole::SetWindow(const RECT& layout) std::swap(buffer, m_buffer); std::swap(lines, m_lines); + Utils::Logf("TextConsole initialized at (%d,%d) - (%d,%d) with %d rows, %d columns\n", + layout.left, layout.top, layout.right, layout.bottom, m_rows, m_columns); + if ((m_currentColumn >= m_columns) || (m_currentLine >= m_rows)) { IncrementLine(); diff --git a/Plot/ImGuiPlots.cpp b/Plot/ImGuiPlots.cpp index b002b9df..3233547d 100644 --- a/Plot/ImGuiPlots.cpp +++ b/Plot/ImGuiPlots.cpp @@ -23,10 +23,10 @@ ImGuiPlots::ImGuiPlots() : Plot(kPlotDescs[PLOT_HOST_FRAMETIME]), Plot(kPlotDescs[PLOT_VSYNC_INTERVAL]), Plot(kPlotDescs[PLOT_DROPPED_NETWORK]), - Plot(kPlotDescs[PLOT_DROPPED_PACER_BACK]), - Plot(kPlotDescs[PLOT_DROPPED_PACER_FRONT]), - Plot(kPlotDescs[PLOT_OVERHEAD]), + Plot(kPlotDescs[PLOT_DROPPED_PACER]), + Plot(kPlotDescs[PLOT_PRESENT_PACING]), + Plot(kPlotDescs[PLOT_OVERHEAD]), Plot(kPlotDescs[PLOT_ETC]), }} { diff --git a/Plot/PlotDesc.h b/Plot/PlotDesc.h index e80f2f0b..3af7b038 100644 --- a/Plot/PlotDesc.h +++ b/Plot/PlotDesc.h @@ -3,28 +3,26 @@ #pragma once #include -enum PlotType -{ +enum PlotType { PLOT_FRAMETIME = 0, PLOT_HOST_FRAMETIME, PLOT_VSYNC_INTERVAL, PLOT_DROPPED_NETWORK, - PLOT_DROPPED_PACER_BACK, - PLOT_DROPPED_PACER_FRONT, + PLOT_DROPPED_PACER, + PLOT_PRESENT_PACING, + PLOT_OVERHEAD, PLOT_ETC, PlotCount }; -enum PlotLabelType -{ +enum PlotLabelType { PLOT_LABEL_MIN_MAX_AVG = 0, PLOT_LABEL_MIN_MAX_AVG_INT, PLOT_LABEL_TOTAL_INT }; -struct PlotDesc -{ +struct PlotDesc { const char *title; PlotLabelType labelType; const char *unit; @@ -36,12 +34,12 @@ struct PlotDesc // clang-format off inline constexpr std::array kPlotDescs = {{ - {"Frametime", PLOT_LABEL_MIN_MAX_AVG, "ms", -0.1f, 50.0f, 0.0f, 49.9f}, - {"Host Frametime", PLOT_LABEL_MIN_MAX_AVG, "ms", -0.1f, 50.0f, 0.0f, 49.9f}, + {"Frametime", PLOT_LABEL_MIN_MAX_AVG, "ms", -0.1f, 50.0f, 0.0f, 41.0f}, + {"Host Frametime", PLOT_LABEL_MIN_MAX_AVG, "ms", -0.1f, 50.0f, 0.0f, 41.0f}, {"Vsync interval", PLOT_LABEL_MIN_MAX_AVG, "ms", -0.1f, 25.0f, 0.0f, 0.0f}, - {"Dropped Frames (network)", PLOT_LABEL_TOTAL_INT, "", -1.0f, 3.0f, 0.0f, 0.0f}, - {"Dropped Frames (back pacing)", PLOT_LABEL_TOTAL_INT, "", -1.0f, 3.0f, 0.0f, 0.0f}, - {"Dropped Frames (front pacing)", PLOT_LABEL_TOTAL_INT, "", -1.0f, 3.0f, 0.0f, 0.0f}, + {"Dropped frames (network)", PLOT_LABEL_TOTAL_INT, "", -1.0f, 3.0f, 0.0f, 0.0f}, + {"Dropped frames (pacing)", PLOT_LABEL_TOTAL_INT, "", -1.0f, 3.0f, 0.0f, 0.0f}, + {"Present to display latency", PLOT_LABEL_MIN_MAX_AVG, "ms", -10.0f, 25.0f, 0.0f, 0.0f}, {"Graph overhead", PLOT_LABEL_MIN_MAX_AVG, "ms", -0.1f, 6.0f, 0.0f, 0.0f}, {"Etc...", PLOT_LABEL_MIN_MAX_AVG, "ms", -0.1f, 50.0f, 0.0f, 49.9f}, }}; diff --git a/State/Stats.cpp b/State/Stats.cpp index 23f16278..a924e3f7 100644 --- a/State/Stats.cpp +++ b/State/Stats.cpp @@ -89,7 +89,7 @@ void Stats::SubmitVideoBytesAndReassemblyTime(uint32_t length, PDECODE_UNIT deco m_ActiveWndVideoStats.networkDroppedFrames += droppedFrames; m_ActiveWndVideoStats.totalFrames += droppedFrames; } - ImGuiPlots::instance().observeFloat(PLOT_DROPPED_NETWORK, droppedFrames); + ImGuiPlots::instance().observeFloat(PLOT_DROPPED_NETWORK, (float)droppedFrames); // Host frametime graph, uses raw 90kHz units to avoid rounding errors static uint32_t lastHostPts = 0; @@ -124,6 +124,12 @@ void Stats::SubmitPacerTime(int64_t pacerTimeQpc, int64_t renderTimeQpc) { m_ActiveWndVideoStats.renderedFrames++; } +// Present to display latency (how close to hitting vblank we are) +void Stats::SubmitPresentPacing(double presentDisplayMs) { + std::lock_guard lock(m_mutex); + m_ActiveWndVideoStats.totalPresentDisplayMs += presentDisplayMs; +} + /// private methods void Stats::addVideoStats(DX::StepTimer const& timer, VIDEO_STATS& src, VIDEO_STATS& dst) { @@ -137,6 +143,7 @@ void Stats::addVideoStats(DX::StepTimer const& timer, VIDEO_STATS& src, VIDEO_ST dst.totalDecodeTime += src.totalDecodeTime; dst.totalPacerTimeUs += src.totalPacerTimeUs; dst.totalRenderTimeUs += src.totalRenderTimeUs; + dst.totalPresentDisplayMs += src.totalPresentDisplayMs; if (dst.minHostProcessingLatency == 0) { dst.minHostProcessingLatency = src.minHostProcessingLatency; @@ -332,7 +339,7 @@ void Stats::formatVideoStats(DX::StepTimer const& timer, VIDEO_STATS& stats, cha "Frames dropped due to network jitter: %.2f%%\n" "Average network latency: %s\n" "Average reassembly/decoding time: %.2f/%.2f ms\n" - "Average frame queue delay: %.2f ms\n" + "Average frame queue/present delay: %.2f/%.2f ms\n" "Average render time: %.2f ms\n", stats.totalFrames ? (double)stats.networkDroppedFrames / stats.totalFrames * 100 : 0.0f, stats.totalFrames ? (double)stats.pacerDroppedFrames / stats.totalFrames * 100 : 0.0f, @@ -340,6 +347,7 @@ void Stats::formatVideoStats(DX::StepTimer const& timer, VIDEO_STATS& stats, cha stats.decodedFrames ? (double)stats.totalReassemblyTime / stats.decodedFrames : 0.0f, stats.decodedFrames ? (double)stats.totalDecodeTime / stats.decodedFrames : 0.0f, stats.renderedFrames ? (double)stats.totalPacerTimeUs / 1000.0 / stats.renderedFrames : 0.0f, + stats.renderedFrames ? (double)stats.totalPresentDisplayMs / stats.renderedFrames : 0.0f, stats.renderedFrames ? (double)stats.totalRenderTimeUs / 1000.0 / stats.renderedFrames : 0.0f); if (ret < 0 || ret >= length - offset) { Utils::Log("Error: stringifyVideoStats length overflow\n"); diff --git a/State/Stats.h b/State/Stats.h index 571ad90a..558b7548 100644 --- a/State/Stats.h +++ b/State/Stats.h @@ -34,6 +34,7 @@ typedef struct _VIDEO_STATS { double totalDecodeTime; uint64_t totalPacerTimeUs; uint64_t totalRenderTimeUs; + double totalPresentDisplayMs; uint32_t lastRtt; uint32_t lastRttVariance; double totalFps; @@ -56,6 +57,7 @@ namespace moonlight_xbox_dx void SubmitDecodeMs(double decodeMs); void SubmitDroppedFrame(int count); void SubmitPacerTime(int64_t pacerTimeQpc, int64_t renderTimeQpc); + void SubmitPresentPacing(double presentDisplayMs); void SetDisplayStatus(SyncMode status) { m_displayStatus = status; } diff --git a/Streaming/FFmpegDecoder.cpp b/Streaming/FFmpegDecoder.cpp index 2cee208b..75bebeb9 100644 --- a/Streaming/FFmpegDecoder.cpp +++ b/Streaming/FFmpegDecoder.cpp @@ -55,8 +55,7 @@ namespace moonlight_xbox_dx { ffmpeg_buffer(nullptr), ffmpeg_buffer_size(0), resources(nullptr), - m_LastFrameNumber(0), - m_Pacer(nullptr) { + m_LastFrameNumber(0) { } void ffmpeg_log_callback(void *ptr, int level, const char *fmt, va_list vl) { @@ -78,9 +77,7 @@ namespace moonlight_xbox_dx { void FFMpegDecoder::CompleteInitialization(const std::shared_ptr& res, STREAM_CONFIGURATION *config) { this->resources = res; - - m_Pacer = std::make_unique(res); - m_Pacer->initialize(config->fps, res->GetRefreshRate()); + Pacer::instance().init(res, config->fps, res->GetRefreshRate()); } int FFMpegDecoder::Init(int videoFormat, int width, int height, int redrawRate, void* context, int drFlags) { @@ -191,7 +188,8 @@ namespace moonlight_xbox_dx { ffmpeg_buffer_size = 0; } m_LastFrameNumber = 0; - m_Pacer.reset(nullptr); + + Pacer::instance().deinit(); Utils::Log("FFMpegDecoder::Cleanup\n"); } @@ -280,7 +278,7 @@ namespace moonlight_xbox_dx { frame_attach_userdata(frame, decodeUnit->presentationTimeMs, decodeEnd.QuadPart); // Queue the frame for rendering (or render now if pacer is disabled) - m_Pacer->submitFrame(frame); + Pacer::instance().submitFrame(frame); double decodeTimeMs = QpcToMs(decodeEnd.QuadPart - decodeStart.QuadPart); FQLog("✓ Frame decoded [pts: %.3f] [in#: %d] [out#: %d] [lost: %d] decode time %.3f ms\n", @@ -305,17 +303,6 @@ namespace moonlight_xbox_dx { return DR_OK; } - int FFMpegDecoder::GetFrameDropTarget() { - if (m_Pacer) { - return m_Pacer->getFrameDropTarget(); - } - return 0; - } - - int FFMpegDecoder::ModifyFrameDropTarget(bool increase) { - return m_Pacer->modifyFrameDropTarget(increase); - } - //Helpers int initCallback(int videoFormat, int width, int height, int redrawRate, void* context, int drFlags) noexcept { return FFMpegDecoder::instance().Init(videoFormat, width, height, redrawRate, context, drFlags); @@ -339,25 +326,4 @@ namespace moonlight_xbox_dx { //decoder_callbacks_sdl.capabilities = CAPABILITY_DIRECT_SUBMIT | CAPABILITY_REFERENCE_FRAME_INVALIDATION_HEVC; return decoder_callbacks_sdl; } - - -} - -void FFMpegDecoder::WaitForFrame() { - if (m_Pacer) { - m_Pacer->waitForFrame(); - } -} - -bool FFMpegDecoder::RenderFrameOnMainThread(std::shared_ptr &sceneRenderer) { - if (m_Pacer) { - return m_Pacer->renderOnMainThread(sceneRenderer); - } - return false; -} - -void FFMpegDecoder::WaitUntilPresentTarget() { - if (m_Pacer) { - m_Pacer->waitUntilPresentTarget(); - } } diff --git a/Streaming/FFmpegDecoder.h b/Streaming/FFmpegDecoder.h index 613dd43f..bbed81ab 100644 --- a/Streaming/FFmpegDecoder.h +++ b/Streaming/FFmpegDecoder.h @@ -19,7 +19,8 @@ extern "C" { typedef struct MLFrameData { uint32_t presentationTimeMs; // host's pts int64_t decodeEndQpc; // when we finished decoding - int64_t presentTargetQpc; // timestamp when frame should be presented + int64_t presentTargetQpc; // timestamp when frame should be presented (slightly earlier than vsync) + int64_t presentVsyncQpc; // hard vsync deadline } MLFrameData; namespace moonlight_xbox_dx { @@ -33,11 +34,6 @@ class FFMpegDecoder { int Init(int videoFormat, int width, int height, int redrawRate, void *context, int drFlags); void Cleanup(); int SubmitDecodeUnit(PDECODE_UNIT decodeUnit); - int GetFrameDropTarget(); - int ModifyFrameDropTarget(bool increase); - void WaitForFrame(); - bool RenderFrameOnMainThread(std::shared_ptr &sceneRenderer); - void WaitUntilPresentTarget(); static FFMpegDecoder *getInstance(); static DECODER_RENDERER_CALLBACKS getDecoder(); std::recursive_mutex mutex; @@ -57,6 +53,5 @@ class FFMpegDecoder { int ffmpeg_buffer_size; std::shared_ptr resources; int m_LastFrameNumber; - std::unique_ptr m_Pacer; }; } // namespace moonlight_xbox_dx diff --git a/Streaming/Pacer.cpp b/Streaming/Pacer.cpp index ed9d3c1f..bd5b58b7 100644 --- a/Streaming/Pacer.cpp +++ b/Streaming/Pacer.cpp @@ -1,49 +1,66 @@ -#include "Pacer.h" +// clang-format off #include "pch.h" +// clang-format on +#include "Pacer.h" #include #include #include #include "../Plot/ImGuiPlots.h" #include "FFmpegDecoder.h" -#include "PacerRational.h" #include "Utils.hpp" // Frame Pacing operation +// +// This code is derived from moonlight-qt's pacer.cpp, and has substantial improvements to deal with Xbox-specific +// challenges. +// // 4 threads use this class: // // Decoder thread (run from moonlight-common-c because DIRECT_SUBMIT) // * submits AVFrame frames to pacing queue via submitFrame() -// * max frames in pacing queue = 4, will drop oldest frame if it's full (dropped by back pacing) +// * max frames in pacing queue = 3, will drop oldest frame if it's full (dropped by back pacing) // // vsyncEmulator thread: -// * Creates our own 120hz vsync signal since Xbox UWP only supports 60hz +// * Creates our own 120hz vsync signal since Xbox UWP only supports 60hz. // * attempts to send m_VsyncSignalled as accurately locked to the refresh rate as possible. // * uses rational integer math to support 59.94 and 119.88 -// * syncs periodically with hardware 60hz vsync to prevent drift +// * alternates 1 frame synced to hardware vblank, a 1 frame on timer +// * since 120hz requires running without vsync, this thread also manages a TearController that schedules +// Present() calls in an attempt to reduce/hide/move the location of the screen tearing line. It sort of +// works, but mostly we're still at the mercy of the DWM Compositor. Xbox Series consoles deal with +// tearing better than Xbox One X. +// +// vsyncHardware thread: +// * Used when streaming at 30 or 60fps. Runs with vsync so there is no screen tearing. // // backPacer thread (called vsyncThread in moonlight-qt) -// * wakes up on each signal from vsync thread, uses same deadline Qpc as emulator thread +// * wakes up on each signal from the vsync thread, uses same deadline Qpc as emulator thread // * maintains pacing queue // * moves 1 frame to render queue -// * tries to accurately time signal to main thread +// * passes along current vblank timestamps to the render thread // * signals main thread to start rendering // // renderOnMainThread // * called from main render loop after waiting on GetFrameLatencyWaitableObject and signal from backPacer // * calls frontPacer() (renderFrame in moonlight-qt) -// * maintains render queue +// * Using vblank timestamps passed down from vsync thread, WaitUntilPresentTarget() schedules precise +// Present() calls, and is somewhat able to control screen tearing location. This also enables butter +// smooth frame pacing. +// * Calls AfterPresent() to capture stats and drift timing information, passed to TearController // -// Other differences from moonlight-qt: +// Other notes and differences from moonlight-qt: // SwapChain BufferCount = 3 // GetFrameLatencyWaitableObject() is used to wait before each frame -// SetMaximumFrameLatency(1) (TODO: with dynamic adjustment to 2 if necessary?) +// SetMaximumFrameLatency(1) +// Calls to FQLog() and functions called within FQLog() are no-op unless you define FRAME_QUEUE_VERBOSE in pch.h +// and build in Debug mode. // Limit the number of queued frames to prevent excessive memory consumption // if the V-Sync source or renderer is blocked for a while. It's important // that the sum of all queued frames between both pacing and rendering queues // must not exceed the number buffer pool size to avoid running the decoder // out of available decoding surfaces. -#define MAX_QUEUED_FRAMES 4 +#define MAX_QUEUED_FRAMES 3 // We may be woken up slightly late so don't go all the way // up to the next V-sync since we may accidentally step into @@ -55,18 +72,25 @@ using steady_clock = std::chrono::steady_clock; using namespace moonlight_xbox_dx; -Pacer::Pacer(const std::shared_ptr &res) : - m_DeviceResources(res), +Pacer &Pacer::instance() { + static Pacer inst; + return inst; +} + +Pacer::Pacer() : + m_Running(false), + m_DeviceResources(), + m_VsyncThread(), m_BackPacerThread(), m_VsyncSeq(0), m_VsyncNextDeadlineQpc(0), m_Stopping(false), m_StreamFps(0), - m_RefreshRate(0.0), - m_FrameDropTarget(0) { + m_RefreshRate(0.0) { } -Pacer::~Pacer() { +void Pacer::deinit() { + m_Running = false; m_Stopping.store(true, std::memory_order_release); m_RenderQueueNotEmpty.notify_all(); @@ -76,10 +100,12 @@ Pacer::~Pacer() { if (m_BackPacerThread.joinable()) { m_BackPacerThread.join(); } - if (m_VsyncEmulator.joinable()) { - m_VsyncEmulator.join(); + if (m_VsyncThread.joinable()) { + m_VsyncThread.join(); } + m_DeviceResources = nullptr; + // Drain and free any remaining frames in both queues { std::unique_lock lock(m_FrameQueueLock); @@ -99,25 +125,29 @@ Pacer::~Pacer() { lock.lock(); } } + + Utils::Logf("Pacer: deinit\n"); } -void Pacer::initialize(int streamFps, double refreshRate) { +void Pacer::init(const std::shared_ptr &res, int streamFps, double refreshRate) { + m_Stopping.store(false, std::memory_order_release); + m_DeviceResources = res; m_StreamFps = streamFps; m_RefreshRate = refreshRate; - Utils::Logf("Frame pacing: target %.2f Hz with %d FPS stream", + Utils::Logf("Pacer: init target %.2f Hz with %d FPS stream", m_RefreshRate, m_StreamFps); // If running on Xbox at 120hz, emulate the vsync interval if (m_DeviceResources->isXbox() && m_StreamFps == 120) { // Start the timing emulator thread - if (!m_VsyncEmulator.joinable()) { - m_VsyncEmulator = std::thread(&Pacer::vsyncEmulator, this); + if (!m_VsyncThread.joinable()) { + m_VsyncThread = std::thread(&Pacer::vsyncEmulator, this); } } else { // Much simpler hardware vsync thread - if (!m_VsyncEmulator.joinable()) { - m_VsyncEmulator = std::thread(&Pacer::vsyncHardware, this); + if (!m_VsyncThread.joinable()) { + m_VsyncThread = std::thread(&Pacer::vsyncHardware, this); } } @@ -125,61 +155,28 @@ void Pacer::initialize(int streamFps, double refreshRate) { if (!m_BackPacerThread.joinable()) { m_BackPacerThread = std::thread(&Pacer::backPacer, this); } + + m_Running = true; } // Vsync Emulator // Sleep until approximately targetQpc, then busy-wait to land precisely. // sleepSlackUs: how early (in microseconds) to stop sleeping and start spinning -// tightSpinUs: within this final window, use a tight YieldProcessor loop -static inline void SleepUntilQpc(int64_t targetQpc, - int64_t sleepSlackUs = 1500, - int64_t tightSpinUs = 150) { +static inline void SleepUntilQpc(int64_t targetQpc, int64_t sleepSlackUs = 1000) { const int64_t f = QpcFreq(); - const int64_t sleepSlackQpc = UsToQpc(sleepSlackUs); - const int64_t tightSpinQpc = UsToQpc(tightSpinUs); - + const int64_t slack = UsToQpc(sleepSlackUs); for (;;) { const int64_t now = QpcNow(); - int64_t remaining = targetQpc - now; + const int64_t remaining = targetQpc - now; if (remaining <= 0) break; - - if (remaining > sleepSlackQpc) { - int64_t toSleepQpc = remaining - sleepSlackQpc; - DWORD ms = (DWORD)((toSleepQpc * 1000) / f); - if (ms == 0) ms = 1; - Sleep(ms); - continue; - } - - if (remaining > tightSpinQpc) { - YieldProcessor(); + if (remaining > slack) { + DWORD ms = (DWORD)(((remaining - slack) * 1000 + f / 2) / f); + if (ms > 0) Sleep(ms); continue; } - - do { - YieldProcessor(); - } while ((targetQpc - QpcNow()) > 0); - break; - } -} - -int64_t Pacer::measureVsyncQpc(int intervals) { - auto *out = m_DeviceResources->GetDXGIOutput(); - - // Align to a vblank edge and measure deltas - out->WaitForVBlank(); - int64_t prev = QpcNow(); - int64_t sum = 0; - - for (int i = 0; i < intervals; ++i) { - out->WaitForVBlank(); - const int64_t now = QpcNow(); - sum += (now - prev); - prev = now; + YieldProcessor(); } - - return sum / intervals; } void Pacer::vsyncEmulator() { @@ -187,7 +184,6 @@ void Pacer::vsyncEmulator() { Utils::Logf("Failed to set vsyncEmulator priority: %d\n", GetLastError()); } - QpcRationalPeriod period; uint32_t hzNum = 60; uint32_t hzDen = 1; @@ -210,57 +206,87 @@ void Pacer::vsyncEmulator() { break; } - // measure system vsync rate so we stay in phase - // I experimented with having this run at 240hz but it doesn't seem necessary - int64_t systemVsyncQpc = measureVsyncQpc(8); - int32_t phase = (int32_t)(QpcToMs(systemVsyncQpc) / 1000.0 / (1.0 / ((double)hzNum / hzDen)) + 0.5); // e.g. 16.6 / 8.3 = 2 + int64_t vsyncQpc = waitForVBlank(); + m_VsyncPeriod.initFromHz(hzNum, hzDen, vsyncQpc); + m_TearCtl.initializeTiming(QpcToMs(m_VsyncPeriod.baseTicks), 0.10); + m_TearCtl.setTearOffset(0.0); // XXX use saved setting - Utils::Logf("Pacer: hardware vsync interval %.2f ms\n", QpcToMs(systemVsyncQpc)); - Utils::Logf("Pacer: using emulated vsync timer @ %.2f Hz using integer rate %lu/%lu\n", - m_RefreshRate, hzNum, hzDen); + Utils::Logf("Pacer: using emulated integer vsync timer %lu/%lu based on system refresh rate %.2f Hz\n", + hzNum, hzDen, m_RefreshRate); - // 30s resync seems about right, on Series X it drifts about 0.3ms in this time - static constexpr int32_t remeasureEveryFrames = 120 * 30; - uint64_t frames = 0; int64_t baseQpc = 0; - int32_t resyncInFrames = 0; + int64_t lastTick = 0; + uint64_t lastVblankSeq = 0; - while (!stopping()) { - int64_t curDeadlineQpc = period.nextDeadlineQpc; + const bool is120 = m_RefreshRate > 60.0; + bool tick = true; // tick = hardware vsync, tock = emulated vsync (120hz) + // tick = hardware vsync only (60hz) - if (resyncInFrames <= 0 && frames % phase == 0) { - // Align to a real vblank edge at startup and at regular intervals - m_DeviceResources->GetDXGIOutput()->WaitForVBlank(); - baseQpc = QpcNow(); - period.initFromHz(hzNum, hzDen, baseQpc); + uint64_t emuClockTTL = 2; // resync with hardware every other frame. This can be any even number. + uint64_t ttl = 0; - if (curDeadlineQpc > 0) { - FQLog("resync to hardware vsync, frame: %d, drift: %lld ticks (%.3f ms), phase: %d\n", - frames, period.nextDeadlineQpc - curDeadlineQpc, QpcToMs(period.nextDeadlineQpc - curDeadlineQpc), phase); + while (!stopping()) { + if (tick) { +#ifdef FRAME_QUEUE_VERBOSE + int64_t pre = QpcNow(); +#endif + // Align to a real vblank edge + baseQpc = waitForVBlank(); + +#ifdef FRAME_QUEUE_VERBOSE + // compare what our emulated deadline would have been with real vblank + // TODO: is there any value in running on the emulated timer for longer periods? + if (m_VsyncSeq > 0) { + int64_t jitterQpc = std::llabs(m_VsyncPeriod.nextDeadlineQpc - baseQpc); + Utils::Logf("emulation deadline jitter after %d cycles: %lld (%f)\n", + m_VsyncSeq - lastVblankSeq, + jitterQpc, QpcToMs(jitterQpc)); } +#endif + + m_VsyncPeriod.initFromHz(hzNum, hzDen, baseQpc); + FQLog("resync to hardware vsync, waited %.3fms\n", QpcToMs(baseQpc - pre)); + lastVblankSeq = m_VsyncSeq; - curDeadlineQpc = baseQpc; - resyncInFrames = remeasureEveryFrames; + if (is120) { + // when running at 120hz, next vsync is emulated + tick = false; + ttl = emuClockTTL - 1; + } } else { +#ifdef FRAME_QUEUE_VERBOSE + int64_t pre = QpcNow(); +#endif // Emulate the vsync interval - SleepUntilQpc(period.nextDeadlineQpc); - --resyncInFrames; + SleepUntilQpc(m_VsyncPeriod.nextDeadlineQpc); + FQLog("waited %.3fms for emulated vsync\n", QpcToMs(QpcNow() - pre)); + + if (--ttl == 0) { + // time for a hardware vsync + tick = true; + } } // Compute next period - period.step(); - ++frames; + m_VsyncPeriod.step(); // signal vsync thread, passing it the next deadline { std::lock_guard lock(m_VsyncMutex); ++m_VsyncSeq; - m_VsyncNextDeadlineQpc = period.nextDeadlineQpc; + m_VsyncNextDeadlineQpc = m_VsyncPeriod.nextDeadlineQpc; } m_VsyncSignalled.notify_one(); - ImGuiPlots::instance().observeFloat(PLOT_VSYNC_INTERVAL, (float)QpcToMs(period.nextDeadlineQpc - curDeadlineQpc)); + int64_t nowTick = QpcNow(); + if (lastTick > 0) { + FQLog("vsync interval %.3f ms\n", QpcToMs(nowTick - lastTick)); + ImGuiPlots::instance().observeFloat(PLOT_VSYNC_INTERVAL, (float)QpcToMs(nowTick - lastTick)); + } + lastTick = nowTick; } + + Utils::Logf("vsyncEmulator thread stopped\n"); } void Pacer::vsyncHardware() { @@ -268,22 +294,56 @@ void Pacer::vsyncHardware() { Utils::Logf("Failed to set vsyncHardware priority: %d\n", GetLastError()); } - // measure system vsync rate - int64_t systemVsyncQpc = measureVsyncQpc(8); + // The hardware vsync path can be called for the following cases: + // system 120hz, stream 60 or 30fps + // system 60hz, stream 60 or 30fps + // Same but with 119.88hz/59.94hz + + uint32_t hzNum = 60; + uint32_t hzDen = 1; + + // Detect if we need special NTSC handling + switch ((int)std::round(m_RefreshRate * 100)) { + case 5994: + case 11988: + hzNum = 60000; + hzDen = 1001; + break; + case 6000: + case 12000: + hzNum = 60; + hzDen = 1; + break; + default: + hzNum = m_StreamFps; + hzDen = 1; + } + int64_t lastT0 = 0; - Utils::Logf("Pacer: using hardware vsync interval %.2f ms\n", QpcToMs(systemVsyncQpc)); + // no tearing to worry about, we will just use it for timing + int64_t vsyncQpc = waitForVBlank(); + m_VsyncPeriod.initFromHz(hzNum, hzDen, vsyncQpc); + m_TearCtl.initializeTiming(QpcToMs(m_VsyncPeriod.baseTicks), 0.10); + m_TearCtl.setTearOffset(0.0); //; always 0.0 in vsync mode + + Utils::Logf("Pacer: using integer vsync timer %lu/%lu based on system refresh rate %.2f Hz\n", + hzNum, hzDen, m_RefreshRate); while (!stopping()) { // Align to a real vblank - m_DeviceResources->GetDXGIOutput()->WaitForVBlank(); - const int64_t now = QpcNow(); + int64_t now = waitForVBlank(); + + m_VsyncPeriod.initFromHz(hzNum, hzDen, now); + + // Compute next period + m_VsyncPeriod.step(); // signal vsync thread, passing it the next deadline { std::lock_guard lock(m_VsyncMutex); ++m_VsyncSeq; - m_VsyncNextDeadlineQpc = now + systemVsyncQpc; + m_VsyncNextDeadlineQpc = m_VsyncPeriod.nextDeadlineQpc; } m_VsyncSignalled.notify_one(); @@ -292,6 +352,8 @@ void Pacer::vsyncHardware() { } lastT0 = now; } + + Utils::Logf("vsyncHardware thread stopped\n"); } void Pacer::backPacer() { @@ -319,12 +381,15 @@ void Pacer::backPacer() { handleVsync(nextDeadlineQpc); - LARGE_INTEGER t0; - static int64_t lastT0 = 0; - QueryPerformanceCounter(&t0); - FQLog("backPacer interval %.3f ms\n", QpcToMs(t0.QuadPart - lastT0)); - lastT0 = t0.QuadPart; +#ifdef FRAME_QUEUE_VERBOSE + int64_t now = QpcNow(); + static int64_t last = 0; + FQLog("backPacer interval %.3f ms\n", QpcToMs(now - last)); + last = now; +#endif } + + Utils::Logf("backPacer thread stopped\n"); } void Pacer::handleVsync(int64_t nextDeadlineQpc) { @@ -368,7 +433,7 @@ void Pacer::handleVsync(int64_t nextDeadlineQpc) { int dropCount = 0; while (m_PacingQueue.size() > frameDropTarget) { ++dropCount; - AVFrame *frame = std::move(m_PacingQueue.front()); + AVFrame *frame = m_PacingQueue.front(); m_PacingQueue.pop_front(); FQLog("! pacing queue overflow: %d > frame drop target %d, dropping oldest frame (pts %.3f ms)\n", @@ -383,16 +448,17 @@ void Pacer::handleVsync(int64_t nextDeadlineQpc) { // Don't graph 0 here, because it's already being done in dropFrameForEnqueue if (dropCount > 0) { - ImGuiPlots::instance().observeFloat(PLOT_DROPPED_PACER_BACK, (float)dropCount); + ImGuiPlots::instance().observeFloat(PLOT_DROPPED_PACER, (float)dropCount); } if (m_PacingQueue.empty()) { +#ifdef FRAME_QUEUE_VERBOSE const int64_t t0 = QpcNow(); +#endif m_PacingQueueNotEmpty.wait_for(lock, waitUs, [this] { return stopping() || !m_PacingQueue.empty(); }); - const int64_t t1 = QpcNow(); - FQLog("handleVsync waited for m_PacingQueueNotEmpty for %.3f ms\n", QpcToMs(t1 - t0)); + FQLog("handleVsync waited for m_PacingQueueNotEmpty for %.3f ms\n", QpcToMs(QpcNow() - t0)); if (stopping() || m_PacingQueue.empty()) { return; @@ -402,13 +468,16 @@ void Pacer::handleVsync(int64_t nextDeadlineQpc) { } // Place the first frame on the render queue - AVFrame *frame = std::move(m_PacingQueue.front()); + AVFrame *frame = m_PacingQueue.front(); m_PacingQueue.pop_front(); // Pass along the target timestamp with the frame metadata if (frame->opaque_ref) { auto *data = reinterpret_cast(frame->opaque_ref->data); - data->presentTargetQpc = nextDeadlineQpc; + data->presentVsyncQpc = nextDeadlineQpc; + + // Adjust Present() timing + data->presentTargetQpc = m_TearCtl.targetPresentQpc(nextDeadlineQpc); } lock.unlock(); @@ -420,7 +489,7 @@ void Pacer::handleVsync(int64_t nextDeadlineQpc) { void Pacer::enqueueFrameForRenderingAndUnlock(AVFrame *frame) { { std::scoped_lock lock(m_FrameQueueLock); - dropFrameForEnqueue(m_RenderQueue, PLOT_DROPPED_PACER_FRONT); + dropFrameForEnqueue(m_RenderQueue); m_RenderQueue.push_back(frame); } @@ -428,34 +497,39 @@ void Pacer::enqueueFrameForRenderingAndUnlock(AVFrame *frame) { m_RenderQueueNotEmpty.notify_one(); } -// Main thread +// Main render thread void Pacer::waitForFrame() { + if (!m_Running) return; + // Wait for the renderer to be ready for the next frame +#ifdef FRAME_QUEUE_VERBOSE int64_t t0 = QpcNow(); +#endif HANDLE flw = m_DeviceResources->GetFrameLatencyWaitable(); - WaitForSingleObjectEx(flw, 1000, true); + if (WaitForSingleObjectEx(flw, 1000, true) == WAIT_TIMEOUT) { + Utils::Logf("Pacer::waitForFrame(): warning: waited over 1000ms for FrameLatencyWaitable\n"); + } - int64_t t1 = QpcNow(); - FQLog("waitForFrame(): FrameLatencyWaitable waited %.3f ms\n", QpcToMs(t1 - t0)); + FQLog("waitForFrame(): FrameLatencyWaitable waited %.3f ms\n", QpcToMs(QpcNow() - t0)); std::unique_lock lock(m_FrameQueueLock); m_RenderQueueNotEmpty.wait(lock, [this] { return stopping() || !m_RenderQueue.empty(); }); - - int64_t t2 = QpcNow(); - FQLog("waitForFrame(): m_RenderQueueNotEmpty waited %.3f ms\n", QpcToMs(t2 - t1)); } +// called by render thread bool Pacer::renderOnMainThread(std::shared_ptr &sceneRenderer) { + if (!m_Running) return false; + AVFrame *frame = nullptr; { std::scoped_lock lock(m_FrameQueueLock); if (!m_RenderQueue.empty()) { - frame = std::move(m_RenderQueue.front()); + frame = m_RenderQueue.front(); m_RenderQueue.pop_front(); } } @@ -464,39 +538,10 @@ bool Pacer::renderOnMainThread(std::shared_ptr &sceneRenderer) { return false; // no frame, don't Present() } - // Extract the target present time for this frame - if (frame->opaque_ref) { - auto *data = reinterpret_cast(frame->opaque_ref->data); - m_PresentTargetQpc = data ? data->presentTargetQpc : 0; - } else { - m_PresentTargetQpc = 0; - } - frontPacer(sceneRenderer, frame); return true; // ok to Present() } -void Pacer::waitUntilPresentTarget() { - const int64_t target = m_PresentTargetQpc; - if (target <= 0) { - return; - } - - const int64_t now = QpcNow(); - if (target <= now) { - FQLog("waitUntilPresentTarget(): target was %.3f ms too late\n", QpcToMs(now - target)); - return; - } - - FQLog("waitUntilPresentTarget(): waiting for %.3f ms\n", QpcToMs(target - now)); - - SleepUntilQpc(target); - - // Measure how well we timed things - // const double skewMs = QpcToMs(QpcNow() - target); - // ImGuiPlots::instance().observeFloat(PLOT_PRESENT_ACCURACY, (float)skewMs); -} - void Pacer::frontPacer(std::shared_ptr &sceneRenderer, AVFrame *frame) { // Count time spent in Pacer's queues LARGE_INTEGER beforeRender, afterRender; @@ -512,6 +557,12 @@ void Pacer::frontPacer(std::shared_ptr &sceneRenderer, AVFrame *f m_DeviceResources->GetStats()->SubmitPacerTime( beforeRender.QuadPart - data->decodeEndQpc, afterRender.QuadPart - beforeRender.QuadPart); + + // Cache the target present time for this frame + m_PresentTargetQpc = data->presentTargetQpc; + m_PresentVsyncQpc = data->presentVsyncQpc; + } else { + m_PresentTargetQpc = 0; } av_frame_free(&frame); @@ -519,12 +570,12 @@ void Pacer::frontPacer(std::shared_ptr &sceneRenderer, AVFrame *f // Drop frames if we have too many queued up for a while std::unique_lock lock(m_FrameQueueLock); - int frameDropTarget = m_FrameDropTarget; + int frameDropTarget = 0; for (int queueHistoryEntry : m_RenderQueueHistory) { if (queueHistoryEntry == 0) { // Be lenient as long as the queue length // resolves before the end of frame history - frameDropTarget += 2; + frameDropTarget = 2; break; } } @@ -540,7 +591,7 @@ void Pacer::frontPacer(std::shared_ptr &sceneRenderer, AVFrame *f int dropCount = 0; while (m_RenderQueue.size() > frameDropTarget) { ++dropCount; - AVFrame *frame = std::move(m_RenderQueue.front()); + AVFrame *frame = m_RenderQueue.front(); m_RenderQueue.pop_front(); FQLog("! render queue overflow: %d > frame drop target %d, dropping oldest frame (pts %.3f ms)\n", @@ -557,33 +608,81 @@ void Pacer::frontPacer(std::shared_ptr &sceneRenderer, AVFrame *f // Don't graph 0 here, because it's already being done in dropFrameForEnqueue if (dropCount > 0) { - ImGuiPlots::instance().observeFloat(PLOT_DROPPED_PACER_FRONT, (float)dropCount); + ImGuiPlots::instance().observeFloat(PLOT_DROPPED_PACER, (float)dropCount); } } -void Pacer::dropFrameForEnqueue(std::deque &queue, int plotId) { +// called by render thread +void Pacer::waitUntilPresentTarget() { + if (!m_Running) return; + + FQLog("waitUntilPresentTarget(): waiting for %.3fms\n", QpcToMs(m_PresentTargetQpc - QpcNow())); + + SleepUntilQpc(m_PresentTargetQpc); + +#ifdef FRAME_QUEUE_VERBOSE + int64_t now = QpcNow(); + static int64_t last = 0; + if (QpcToMs(now - last) > 5000.0) { + Utils::Logf("wait for Present(): (t %fms) (v %fms)\n", + QpcToMs(now - m_PresentTargetQpc), + QpcToMs(now - m_PresentVsyncQpc)); + last = now; + } +#endif +} + +// called by render thread +void Pacer::afterPresent() { + if (!m_Running) return; + + // Called immediately after Present(), check our timings and adjust future timings + const int64_t now = QpcNow(); + const int64_t driftQpc = m_PresentTargetQpc - now; + double driftMs = QpcToMs(driftQpc); + + { + std::scoped_lock lock(m_FrameQueueLock); + // Measure drift for this frame relative to intended phase. + m_TearCtl.addDriftSample(driftMs); + } + + // technically present-to-slightly-before-display + FQLog("Present-to-present drift %.3fms\n", driftMs); + + // For stats, we show the present-to-vsync latency, which is directly affected by the tear controller + double driftVsyncMs = QpcToMs(m_PresentVsyncQpc - now); + m_DeviceResources->GetStats()->SubmitPresentPacing(driftVsyncMs); + ImGuiPlots::instance().observeFloat(PLOT_PRESENT_PACING, (float)driftVsyncMs); +} + +// end main thread + +void Pacer::dropFrameForEnqueue(std::deque &queue) { int dropCount = 0; if (queue.size() >= MAX_QUEUED_FRAMES) { - AVFrame *frame = std::move(queue.front()); + AVFrame *frame = queue.front(); queue.pop_front(); - FQLog("! queue full (%d), dropping oldest frame (pts %.3f ms)\n", queue.size() + 1, frame->pts / 90.0); m_DeviceResources->GetStats()->SubmitDroppedFrame(1); - av_frame_free(&frame); dropCount = 1; } - ImGuiPlots::instance().observeFloat(plotId, (float)dropCount); + const bool isPacingQueue = (&queue == &m_PacingQueue); + + // when this is zero, only count it for the pacing queue + if (dropCount > 0 || isPacingQueue) { + ImGuiPlots::instance().observeFloat(PLOT_DROPPED_PACER, (float)dropCount); + } } void Pacer::submitFrame(AVFrame *frame) { // Queue the frame and possibly wake up the render thread { std::scoped_lock lock(m_FrameQueueLock); - - dropFrameForEnqueue(m_PacingQueue, PLOT_DROPPED_PACER_BACK); + dropFrameForEnqueue(m_PacingQueue); m_PacingQueue.push_back(frame); } @@ -592,21 +691,21 @@ void Pacer::submitFrame(AVFrame *frame) { m_PacingQueueNotEmpty.notify_one(); } -int Pacer::getFrameDropTarget() { - return m_FrameDropTarget; +// Misc helper functions + +inline int64_t Pacer::waitForVBlank() { + m_DeviceResources->GetDXGIOutput()->WaitForVBlank(); + return QpcNow(); } -int Pacer::modifyFrameDropTarget(bool increase) { - int current = m_FrameDropTarget; - if (increase) { - if (current >= 5) { - return current; - } - return ++m_FrameDropTarget; - } else { - if (current == 0) { - return current; - } - return --m_FrameDropTarget; - } +// passthrough user choice for tear offset +// 0.0 = least latency, present with a short safety buffer +// 100.0 = ~1 frame of latency. +// Somewhere in between may hide screen tearing, or it may not. +void Pacer::setTearOffset(double offset) { + m_TearCtl.setTearOffset(offset); +} + +double Pacer::getTearOffset() { + return m_TearCtl.getTearOffset(); } diff --git a/Streaming/Pacer.h b/Streaming/Pacer.h index 530cc81c..085d3dc8 100644 --- a/Streaming/Pacer.h +++ b/Streaming/Pacer.h @@ -3,6 +3,8 @@ #include #include #include +#include "PacerRational.h" +#include "PacerTearController.h" #include "Utils.hpp" #include "VideoRenderer.h" @@ -12,52 +14,41 @@ extern "C" { class Pacer { public: - Pacer(const std::shared_ptr &res); - - ~Pacer(); - - void submitFrame(AVFrame *frame); - - void initialize(int maxVideoFps, double refreshRate); + // Singleton accessor + static Pacer &instance(); + void deinit(); + void init(const std::shared_ptr &res, int maxVideoFps, double refreshRate); void waitForFrame(); - bool renderOnMainThread(std::shared_ptr &sceneRenderer); - void waitUntilPresentTarget(); - - int getFrameDropTarget(); - - int modifyFrameDropTarget(bool increase); + void afterPresent(); + void setTearOffset(double offset); + double getTearOffset(); + void submitFrame(AVFrame *frame); private: - int64_t measureVsyncQpc(int intervals); + Pacer(); + Pacer(const Pacer &) = delete; + Pacer &operator=(const Pacer &) = delete; - void vsyncEmulator(); + bool stopping() const noexcept { + return m_Stopping.load(std::memory_order_acquire); + } + void vsyncEmulator(); void vsyncHardware(); - void backPacer(); - void handleVsync(int64_t nextDeadlineQpc); - void enqueueFrameForRenderingAndUnlock(AVFrame *frame); - void frontPacer(std::shared_ptr &sceneRenderer, AVFrame *frame); + void dropFrameForEnqueue(std::deque &queue); + int64_t waitForVBlank(); - void dropFrameForEnqueue(std::deque &queue, int plotId); - - bool stopping() const noexcept { - return m_Stopping.load(std::memory_order_acquire); - } - + bool m_Running; std::shared_ptr m_DeviceResources; - std::thread m_VsyncEmulator; + std::thread m_VsyncThread; std::thread m_BackPacerThread; - std::mutex m_VsyncMutex; - std::condition_variable m_VsyncSignalled; - uint64_t m_VsyncSeq; - int64_t m_VsyncNextDeadlineQpc; std::deque m_RenderQueue; std::deque m_PacingQueue; std::deque m_RenderQueueHistory; @@ -65,9 +56,17 @@ class Pacer { std::mutex m_FrameQueueLock; std::condition_variable m_RenderQueueNotEmpty; std::condition_variable m_PacingQueueNotEmpty; - int64_t m_PresentTargetQpc; std::atomic m_Stopping{false}; int m_StreamFps; double m_RefreshRate; - std::atomic_int m_FrameDropTarget; + + QpcRationalPeriod m_VsyncPeriod; + int64_t m_VsyncNextDeadlineQpc; + int64_t m_PresentTargetQpc; + int64_t m_PresentVsyncQpc; + std::mutex m_VsyncMutex; + std::condition_variable m_VsyncSignalled; + uint64_t m_VsyncSeq; + + TearController m_TearCtl; }; diff --git a/Streaming/PacerRational.h b/Streaming/PacerRational.h index 4d1a90dc..ed6a4507 100644 --- a/Streaming/PacerRational.h +++ b/Streaming/PacerRational.h @@ -31,7 +31,7 @@ struct QpcRationalPeriod { remainder = r; den = hzNum; accum = 0; - nextDeadlineQpc = baseQpc + baseTicks; + nextDeadlineQpc = baseQpc; } // Advance to the next deadline diff --git a/Streaming/PacerTearController.h b/Streaming/PacerTearController.h new file mode 100644 index 00000000..d2027da6 --- /dev/null +++ b/Streaming/PacerTearController.h @@ -0,0 +1,100 @@ +#pragma once +#include +#include +#include + +class EmaBiasCorrector { + public: + explicit EmaBiasCorrector(double halfLifeSamples = 60.0, + double targetEarlyMs = 0.05, + double maxPerFrameStepMs = 0.20, + double clampInputMs = 1.0) + : targetEarlyMs_(targetEarlyMs), + maxPerFrameStepMs_(std::max(0.0, maxPerFrameStepMs)), + clampInputMs_(std::max(0.0, clampInputMs)) { + setHalfLife(halfLifeSamples); + } + + void setHalfLife(double halfLifeSamples) { + if (halfLifeSamples <= 1.0) + alpha_ = 1.0; + else + alpha_ = 1.0 - std::pow(2.0, -1.0 / halfLifeSamples); + } + + void addSample(double driftMs) { + const double x = std::clamp(driftMs, -clampInputMs_, clampInputMs_); + if (!has_) { + ema_ = x; + has_ = true; + } else { + ema_ += alpha_ * (x - ema_); + } + const double raw = ema_ - targetEarlyMs_; + correctionMs_ = std::clamp(raw, -maxPerFrameStepMs_, +maxPerFrameStepMs_); + } + + double correctionMs() const { + return has_ ? correctionMs_ : 0.0; + } + double emaMs() const { + return has_ ? ema_ : 0.0; + } + + private: + double alpha_ = 0.1; + double ema_ = 0.0; + double correctionMs_ = 0.0; + double targetEarlyMs_ = 0.05; + double maxPerFrameStepMs_ = 0.25; + double clampInputMs_ = 2.0; + bool has_ = false; +}; + +class TearController { + public: + explicit TearController(double desiredTearFraction = 0.0, + double vblankPeriodMs = 0.0, + double safetyOffsetMs = 0.02) + : ready_(false), + desiredTearFraction_(desiredTearFraction), + vblankPeriodMs_(vblankPeriodMs), + safetyOffsetMs_(std::max(0.0, safetyOffsetMs)), + qpcPerMs_(UsToQpc(1000)) { + } + + void initializeTiming(double vblankPeriodMs, double safetyOffsetMs) { + vblankPeriodMs_ = vblankPeriodMs; + safetyOffsetMs_ = safetyOffsetMs; + ready_ = true; + } + + void setTearOffset(double f) { + desiredTearFraction_.store(std::clamp(f, 0.0, 1.0), std::memory_order_release); + } + + double getTearOffset() const { + return desiredTearFraction_.load(std::memory_order_acquire); + } + + int64_t targetPresentQpc(int64_t nextVblankQpc) const { + if (!ready_) { + throw std::runtime_error("TearController is not initialized"); + } + const double offsetMs = desiredTearFraction_.load(std::memory_order_acquire) * vblankPeriodMs_ + safetyOffsetMs_ + ema_.correctionMs(); + const double clamped = std::clamp(offsetMs, 0.0, std::max(0.0, vblankPeriodMs_ - 0.10)); // make sure we don't push into next vblank + return nextVblankQpc + (int64_t)std::llround(clamped * qpcPerMs_); + } + + void addDriftSample(double driftMs) { + ema_.addSample(driftMs); + } + + private: + bool ready_; + std::atomic desiredTearFraction_; + double vblankPeriodMs_; + double safetyOffsetMs_; + int64_t qpcPerMs_; + EmaBiasCorrector ema_; +}; diff --git a/Streaming/StatsRenderer.cpp b/Streaming/StatsRenderer.cpp index 0f22adbe..9de91543 100644 --- a/Streaming/StatsRenderer.cpp +++ b/Streaming/StatsRenderer.cpp @@ -17,16 +17,14 @@ StatsRenderer::StatsRenderer(const std::shared_ptr &deviceR m_deviceResources(deviceResources), m_mutex(), m_visible(false), - m_stats(stats) -{ + m_stats(stats) { m_console->SetForegroundColor(Colors::Yellow); // m_console->SetDebugOutput(true); CreateDeviceDependentResources(); } -void StatsRenderer::Update(DX::StepTimer const &timer) -{ +void StatsRenderer::Update(DX::StepTimer const &timer) { // We let the Stats class always process even if not visible. Most of the time // it will simply accumulate stats during its 1-second window period. Each second, // when it determines the user-visible text should be updated, it will update outputStr and return true. @@ -43,8 +41,7 @@ void StatsRenderer::Update(DX::StepTimer const &timer) } // Renders a frame to the screen. -void StatsRenderer::Render(bool showImGui) -{ +void StatsRenderer::Render(bool showImGui) { std::lock_guard lock(m_mutex); if (m_visible) { m_console->Render(); @@ -54,23 +51,19 @@ void StatsRenderer::Render(bool showImGui) } } -struct clampData -{ +struct clampData { const float *values; float maxVal; }; -static inline float clampGetter(void *data, int idx) -{ +static inline float clampGetter(void *data, int idx) { const clampData *c = static_cast(data); return fminf(c->values[idx], c->maxVal); } -void StatsRenderer::RenderGraphs() -{ +void StatsRenderer::RenderGraphs() { // Track graph CPU overhead per frame - LARGE_INTEGER t0; - QueryPerformanceCounter(&t0); + int64_t plotStartQpc = QpcNow(); // we malloc a buffer for each stat only once and reuse it each frame assert(PlotCount == 8); @@ -81,14 +74,14 @@ void StatsRenderer::RenderGraphs() (float *)malloc(sizeof(float) * 512), (float *)malloc(sizeof(float) * 512), (float *)malloc(sizeof(float) * 512), - (float *)malloc(sizeof(float) * 512), - (float *)malloc(sizeof(float) * 512)}; + (float *)malloc(sizeof(float) * 512), + (float *)malloc(sizeof(float) * 512)}; ImGuiIO &io = ImGui::GetIO(); float graphW = 850.0f * (m_displayWidth / 3840.0f); float graphH = 120.0f * (m_displayHeight / 2160.0f); - float opacity = 0.7f; + float opacity = 0.8f; LogOnce("Drawing graphs of size %.1f x %.1f in viewport %d x %d using opacity %.2f\n", graphW, graphH, m_displayWidth, m_displayHeight, opacity); @@ -117,7 +110,7 @@ void StatsRenderer::RenderGraphs() float minY = 0.0f; float maxY = 0.0f; - size_t countF = plot.buffer.copyInto(buffers[i], 512, minY, maxY); + int countF = plot.buffer.copyInto(buffers[i], 512, minY, maxY); float avgF = plot.buffer.average(); if (!countF) { return; @@ -161,36 +154,29 @@ void StatsRenderer::RenderGraphs() ImGui::PopID(); }; - const int row1[3] = {PLOT_FRAMETIME, PLOT_DROPPED_NETWORK, PLOT_VSYNC_INTERVAL}; + const int row1[3] = {PLOT_FRAMETIME, PLOT_VSYNC_INTERVAL, PLOT_DROPPED_NETWORK}; for (int c = 0; c < 3; ++c) { if (c > 0) ImGui::SameLine(0.0f, itemSpacingX); draw_plot(row1[c], graphW, graphH); } ImGui::Dummy(ImVec2(1.0f, itemSpacingY)); - const int row2[4] = {PLOT_HOST_FRAMETIME, PLOT_DROPPED_PACER_FRONT, PLOT_DROPPED_PACER_BACK, PLOT_OVERHEAD}; + const int row2[4] = {PLOT_HOST_FRAMETIME, PLOT_PRESENT_PACING, PLOT_DROPPED_PACER}; for (int c = 0; c < 4; ++c) { - float width = graphW; - if (row2[c] == PLOT_DROPPED_PACER_FRONT || row2[c] == PLOT_DROPPED_PACER_BACK) { - width /= 2; - } if (c > 0) ImGui::SameLine(0.0f, itemSpacingX); - draw_plot(row2[c], width, graphH); + draw_plot(row2[c], graphW, graphH); } // 3rd row for quickly graphing something if needed // ImGui::Dummy(ImVec2(1.0f, itemSpacingY)); - // draw_plot(PLOT_DROPPED_PACER_BACK, graphW, graphH); + // draw_plot(PLOT_ETC, graphW, graphH); ImGui::End(); - LARGE_INTEGER t1; - QueryPerformanceCounter(&t1); - ImGuiPlots::instance().observeFloat(PLOT_OVERHEAD, (float)QpcToMs(t1.QuadPart - t0.QuadPart)); + ImGuiPlots::instance().observeFloat(PLOT_OVERHEAD, (float)QpcToMs(QpcNow() - plotStartQpc)); } -void StatsRenderer::CreateDeviceDependentResources() -{ +void StatsRenderer::CreateDeviceDependentResources() { m_deviceResources->GetUWPPixelDimensions(&m_displayWidth, &m_displayHeight); const wchar_t *font = L"Assets\\Font\\ModeSeven-24.spritefont"; // sized for 4K @@ -204,21 +190,34 @@ void StatsRenderer::CreateDeviceDependentResources() m_console->SetFixedWidthFont(true); } -void StatsRenderer::CreateWindowSizeDependentResources() -{ +void StatsRenderer::CreateWindowSizeDependentResources() { + int left = 10; + int right = m_displayWidth / 3; + int bottom = 0; + + if (m_displayHeight >= 2160) { // 24pt font + left = 20; + right = m_displayWidth / 2; + bottom = 448; + } else if (m_displayHeight >= 1440) { // 12pt font + left = 14; + bottom = 224; + } else { + left = 10; + bottom = 168; + } + // The size of our text area (left, top, right, bottom) - RECT size = {20, 0, m_displayWidth * 0.5, m_displayHeight * 0.2}; + RECT size = {left, left, right, bottom}; m_console->SetWindow(size); } -void StatsRenderer::ReleaseDeviceDependentResources() -{ +void StatsRenderer::ReleaseDeviceDependentResources() { m_console->ReleaseDevice(); } -void StatsRenderer::ToggleVisible() -{ +void StatsRenderer::ToggleVisible() { std::lock_guard lock(m_mutex); m_visible = m_visible == true ? false : true; } diff --git a/Streaming/moonlight_xbox_dxMain.cpp b/Streaming/moonlight_xbox_dxMain.cpp index 454e2b4e..6cb0a5d1 100644 --- a/Streaming/moonlight_xbox_dxMain.cpp +++ b/Streaming/moonlight_xbox_dxMain.cpp @@ -121,10 +121,12 @@ void moonlight_xbox_dxMain::StartRenderLoop() Update(); if (Render()) { - FFMpegDecoder::instance().WaitUntilPresentTarget(); + Pacer::instance().waitUntilPresentTarget(); m_deviceResources->Present(); + Pacer::instance().afterPresent(); + int64_t afterPresent = QpcNow(); if (lastPresentTime > 0) { ImGuiPlots::instance().observeFloat(PLOT_FRAMETIME, (float)QpcToMs(afterPresent - lastPresentTime)); @@ -394,8 +396,8 @@ bool moonlight_xbox_dxMain::Render() } // Render the scene objects. - FFMpegDecoder::instance().WaitForFrame(); - bool shouldPresent = FFMpegDecoder::instance().RenderFrameOnMainThread(m_sceneRenderer); + Pacer::instance().waitForFrame(); + bool shouldPresent = Pacer::instance().renderOnMainThread(m_sceneRenderer); m_LogRenderer->Render(); m_statsTextRenderer->Render(showImGui); @@ -535,3 +537,11 @@ void moonlight_xbox_dxMain::SetImGui(bool isVisible) { Utils::Logf("ImGui graphs are now %d\n", isVisible ? 1 : 0); }); } + +void moonlight_xbox_dxMain::SetTearOffset(double offset) { + Pacer::instance().setTearOffset(offset); +} + +double moonlight_xbox_dxMain::GetTearOffset() { + return Pacer::instance().getTearOffset(); +} diff --git a/Streaming/moonlight_xbox_dxMain.h b/Streaming/moonlight_xbox_dxMain.h index abd9ce0b..591d820c 100644 --- a/Streaming/moonlight_xbox_dxMain.h +++ b/Streaming/moonlight_xbox_dxMain.h @@ -39,6 +39,8 @@ namespace moonlight_xbox_dx bool ToggleLogs(); bool ToggleStats(); void SetImGui(bool isVisible); + void SetTearOffset(double offset); + double GetTearOffset(); private: void ProcessInput(); void Update(); diff --git a/Utils/FloatBuffer.cpp b/Utils/FloatBuffer.cpp index 883fc588..b612edc0 100644 --- a/Utils/FloatBuffer.cpp +++ b/Utils/FloatBuffer.cpp @@ -53,7 +53,7 @@ void FloatBuffer::push(float value) noexcept } } -std::size_t FloatBuffer::copyInto(float *outBuffer, std::size_t outSize, float &out_min, float &out_max) const +int FloatBuffer::copyInto(float *outBuffer, std::size_t outSize, float &out_min, float &out_max) const { std::lock_guard lock(mtx_); diff --git a/Utils/FloatBuffer.h b/Utils/FloatBuffer.h index 2af290a7..3c2a5cf2 100644 --- a/Utils/FloatBuffer.h +++ b/Utils/FloatBuffer.h @@ -12,7 +12,7 @@ class FloatBuffer explicit FloatBuffer(std::size_t capacity = kDefaultCapacity); void push(float value) noexcept; - std::size_t copyInto(float *outBuffer, std::size_t outSize, float &out_min, float &out_max) const; + int copyInto(float *outBuffer, std::size_t outSize, float &out_min, float &out_max) const; void clear() noexcept; std::size_t capacity() const noexcept diff --git a/pch.h b/pch.h index afe813ff..2dbd21a3 100644 --- a/pch.h +++ b/pch.h @@ -44,34 +44,48 @@ // Time helpers static inline int64_t QpcFreq() { - static int64_t f = []{ - LARGE_INTEGER li{}; - QueryPerformanceFrequency(&li); - return li.QuadPart; - }(); - return f; + static int64_t f = [] { + LARGE_INTEGER li{}; + QueryPerformanceFrequency(&li); + return li.QuadPart; + }(); + return f; } static inline int64_t QpcNow() { - LARGE_INTEGER li{}; - QueryPerformanceCounter(&li); - return li.QuadPart; + LARGE_INTEGER li{}; + QueryPerformanceCounter(&li); + return li.QuadPart; } static inline int64_t UsToQpc(int64_t us) { - const int64_t f = QpcFreq(); - return (us / UINT64_C(1000000)) * f + - (us % UINT64_C(1000000)) * f / UINT64_C(1000000); + const int64_t f = QpcFreq(); + return (us / INT64_C(1000000)) * f + + (us % INT64_C(1000000)) * f / INT64_C(1000000); } static inline int64_t QpcToUs(int64_t qpc) { const int64_t f = QpcFreq(); - return (qpc / f) * UINT64_C(1000000) + - (qpc % f) * UINT64_C(1000000) / f; + int64_t q = qpc / f; + int64_t r = qpc % f; + if (r < 0) { + --q; + r += f; + } + return q * INT64_C(1000000) + (r * INT64_C(1000000)) / f; +} + +static inline double QpcToMsD(double qpc) { + return qpc * 1000.0 / (double)QpcFreq(); } static inline double QpcToMs(int64_t qpc) { - return (double)QpcToUs(qpc) / 1000.0; + return QpcToMsD(static_cast(qpc)); +} + +static inline int64_t MsToQpc(double ms) { + const int64_t us = static_cast(ms * 1000.0 + 0.5); + return UsToQpc(us); } // Log something only once, safe to use in hot areas of the code @@ -85,14 +99,17 @@ static inline double QpcToMs(int64_t qpc) { }); \ } while (0) -// Frame queue debugging +// Frame queue debugging, uncomment FRAME_QUEUE_VERBOSE +// Note: When FQLog is enabled, the spam is intense, so it only logs data for a short time #if !defined(NDEBUG) //#define FRAME_QUEUE_VERBOSE +static int __once = 0; #endif #ifdef FRAME_QUEUE_VERBOSE #define FQLog(fmt, ...) \ - moonlight_xbox_dx::Utils::Logf("[%d] " fmt, ::GetCurrentThreadId(), ##__VA_ARGS__) + if (++__once > 200 && __once < 1000) \ + moonlight_xbox_dx::Utils::Logf("[%d] " fmt, ::GetCurrentThreadId(), ##__VA_ARGS__) #else #if defined(_MSC_VER) #define FQLog(...) __noop From c9fcb956d495055053492700a95ee4dd0ccb7545 Mon Sep 17 00:00:00 2001 From: Andy Grundman Date: Thu, 16 Oct 2025 13:36:11 -0400 Subject: [PATCH 20/22] Pacer: fix 30fps --- Streaming/Pacer.cpp | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/Streaming/Pacer.cpp b/Streaming/Pacer.cpp index bd5b58b7..18c4c06c 100644 --- a/Streaming/Pacer.cpp +++ b/Streaming/Pacer.cpp @@ -319,6 +319,10 @@ void Pacer::vsyncHardware() { hzDen = 1; } + if (m_StreamFps == 30) { + hzNum /= 2; + } + int64_t lastT0 = 0; // no tearing to worry about, we will just use it for timing @@ -330,10 +334,18 @@ void Pacer::vsyncHardware() { Utils::Logf("Pacer: using integer vsync timer %lu/%lu based on system refresh rate %.2f Hz\n", hzNum, hzDen, m_RefreshRate); + bool skipNextVblank = false; + while (!stopping()) { // Align to a real vblank int64_t now = waitForVBlank(); + // If we're streaming 30fps, wait 2 vblanks per frame + if (m_StreamFps == 30 && skipNextVblank) { + skipNextVblank = false; + continue; + } + m_VsyncPeriod.initFromHz(hzNum, hzDen, now); // Compute next period @@ -351,6 +363,10 @@ void Pacer::vsyncHardware() { ImGuiPlots::instance().observeFloat(PLOT_VSYNC_INTERVAL, (float)QpcToMs(now - lastT0)); } lastT0 = now; + + if (m_StreamFps == 30 && !skipNextVblank) { + skipNextVblank = true; + } } Utils::Logf("vsyncHardware thread stopped\n"); From 48d613778c9052a51e82bc233b7c5608d1d6accd Mon Sep 17 00:00:00 2001 From: Andy Grundman Date: Wed, 29 Oct 2025 04:00:43 -0400 Subject: [PATCH 21/22] Simplify Pacer's present timing --- Plot/PlotDesc.h | 2 +- Streaming/Pacer.cpp | 83 +++++++---------------- Streaming/Pacer.h | 8 +-- Streaming/PacerTearController.h | 100 ---------------------------- Streaming/moonlight_xbox_dxMain.cpp | 11 +-- Streaming/moonlight_xbox_dxMain.h | 2 - 6 files changed, 29 insertions(+), 177 deletions(-) delete mode 100644 Streaming/PacerTearController.h diff --git a/Plot/PlotDesc.h b/Plot/PlotDesc.h index 3af7b038..e52b1f69 100644 --- a/Plot/PlotDesc.h +++ b/Plot/PlotDesc.h @@ -36,7 +36,7 @@ struct PlotDesc { inline constexpr std::array kPlotDescs = {{ {"Frametime", PLOT_LABEL_MIN_MAX_AVG, "ms", -0.1f, 50.0f, 0.0f, 41.0f}, {"Host Frametime", PLOT_LABEL_MIN_MAX_AVG, "ms", -0.1f, 50.0f, 0.0f, 41.0f}, - {"Vsync interval", PLOT_LABEL_MIN_MAX_AVG, "ms", -0.1f, 25.0f, 0.0f, 0.0f}, + {"Vsync interval", PLOT_LABEL_MIN_MAX_AVG, "ms", -0.1f, 50.0f, 0.0f, 0.0f}, {"Dropped frames (network)", PLOT_LABEL_TOTAL_INT, "", -1.0f, 3.0f, 0.0f, 0.0f}, {"Dropped frames (pacing)", PLOT_LABEL_TOTAL_INT, "", -1.0f, 3.0f, 0.0f, 0.0f}, {"Present to display latency", PLOT_LABEL_MIN_MAX_AVG, "ms", -10.0f, 25.0f, 0.0f, 0.0f}, diff --git a/Streaming/Pacer.cpp b/Streaming/Pacer.cpp index 18c4c06c..0a3ba68f 100644 --- a/Streaming/Pacer.cpp +++ b/Streaming/Pacer.cpp @@ -77,16 +77,17 @@ Pacer &Pacer::instance() { return inst; } -Pacer::Pacer() : - m_Running(false), - m_DeviceResources(), - m_VsyncThread(), - m_BackPacerThread(), - m_VsyncSeq(0), - m_VsyncNextDeadlineQpc(0), - m_Stopping(false), - m_StreamFps(0), - m_RefreshRate(0.0) { +Pacer::Pacer() + : m_Running(false), + m_DeviceResources(), + m_VsyncThread(), + m_BackPacerThread(), + m_VsyncSeq(0), + m_VsyncNextDeadlineQpc(0), + m_Stopping(false), + m_StreamFps(0), + m_RefreshRate(0.0), + m_AvgDriftMs(1.2) { } void Pacer::deinit() { @@ -208,8 +209,6 @@ void Pacer::vsyncEmulator() { int64_t vsyncQpc = waitForVBlank(); m_VsyncPeriod.initFromHz(hzNum, hzDen, vsyncQpc); - m_TearCtl.initializeTiming(QpcToMs(m_VsyncPeriod.baseTicks), 0.10); - m_TearCtl.setTearOffset(0.0); // XXX use saved setting Utils::Logf("Pacer: using emulated integer vsync timer %lu/%lu based on system refresh rate %.2f Hz\n", hzNum, hzDen, m_RefreshRate); @@ -233,17 +232,6 @@ void Pacer::vsyncEmulator() { // Align to a real vblank edge baseQpc = waitForVBlank(); -#ifdef FRAME_QUEUE_VERBOSE - // compare what our emulated deadline would have been with real vblank - // TODO: is there any value in running on the emulated timer for longer periods? - if (m_VsyncSeq > 0) { - int64_t jitterQpc = std::llabs(m_VsyncPeriod.nextDeadlineQpc - baseQpc); - Utils::Logf("emulation deadline jitter after %d cycles: %lld (%f)\n", - m_VsyncSeq - lastVblankSeq, - jitterQpc, QpcToMs(jitterQpc)); - } -#endif - m_VsyncPeriod.initFromHz(hzNum, hzDen, baseQpc); FQLog("resync to hardware vsync, waited %.3fms\n", QpcToMs(baseQpc - pre)); lastVblankSeq = m_VsyncSeq; @@ -328,8 +316,6 @@ void Pacer::vsyncHardware() { // no tearing to worry about, we will just use it for timing int64_t vsyncQpc = waitForVBlank(); m_VsyncPeriod.initFromHz(hzNum, hzDen, vsyncQpc); - m_TearCtl.initializeTiming(QpcToMs(m_VsyncPeriod.baseTicks), 0.10); - m_TearCtl.setTearOffset(0.0); //; always 0.0 in vsync mode Utils::Logf("Pacer: using integer vsync timer %lu/%lu based on system refresh rate %.2f Hz\n", hzNum, hzDen, m_RefreshRate); @@ -492,8 +478,12 @@ void Pacer::handleVsync(int64_t nextDeadlineQpc) { auto *data = reinterpret_cast(frame->opaque_ref->data); data->presentVsyncQpc = nextDeadlineQpc; - // Adjust Present() timing - data->presentTargetQpc = m_TearCtl.targetPresentQpc(nextDeadlineQpc); + // Adjust Present() timing, goal is to present as close to nextDeadline without going over + static const int64_t safetyQpc = MsToQpc(0.3); + data->presentTargetQpc = nextDeadlineQpc - MsToQpc(m_AvgDriftMs) - safetyQpc; + if (data->presentTargetQpc > nextDeadlineQpc) { + data->presentTargetQpc = nextDeadlineQpc - safetyQpc; + } } lock.unlock(); @@ -635,41 +625,28 @@ void Pacer::waitUntilPresentTarget() { FQLog("waitUntilPresentTarget(): waiting for %.3fms\n", QpcToMs(m_PresentTargetQpc - QpcNow())); SleepUntilQpc(m_PresentTargetQpc); - -#ifdef FRAME_QUEUE_VERBOSE - int64_t now = QpcNow(); - static int64_t last = 0; - if (QpcToMs(now - last) > 5000.0) { - Utils::Logf("wait for Present(): (t %fms) (v %fms)\n", - QpcToMs(now - m_PresentTargetQpc), - QpcToMs(now - m_PresentVsyncQpc)); - last = now; - } -#endif } // called by render thread -void Pacer::afterPresent() { +void Pacer::afterPresent(int64_t presentTimeQpc) { if (!m_Running) return; // Called immediately after Present(), check our timings and adjust future timings - const int64_t now = QpcNow(); - const int64_t driftQpc = m_PresentTargetQpc - now; + const int64_t driftQpc = presentTimeQpc - m_PresentVsyncQpc; double driftMs = QpcToMs(driftQpc); { std::scoped_lock lock(m_FrameQueueLock); - // Measure drift for this frame relative to intended phase. - m_TearCtl.addDriftSample(driftMs); + // weighted moving average of drift + const double alpha = 0.05f; + m_AvgDriftMs = (driftMs * alpha) + (m_AvgDriftMs * (1.0 - alpha)); } // technically present-to-slightly-before-display FQLog("Present-to-present drift %.3fms\n", driftMs); - // For stats, we show the present-to-vsync latency, which is directly affected by the tear controller - double driftVsyncMs = QpcToMs(m_PresentVsyncQpc - now); - m_DeviceResources->GetStats()->SubmitPresentPacing(driftVsyncMs); - ImGuiPlots::instance().observeFloat(PLOT_PRESENT_PACING, (float)driftVsyncMs); + m_DeviceResources->GetStats()->SubmitPresentPacing(driftMs); + ImGuiPlots::instance().observeFloat(PLOT_PRESENT_PACING, (float)driftMs); } // end main thread @@ -713,15 +690,3 @@ inline int64_t Pacer::waitForVBlank() { m_DeviceResources->GetDXGIOutput()->WaitForVBlank(); return QpcNow(); } - -// passthrough user choice for tear offset -// 0.0 = least latency, present with a short safety buffer -// 100.0 = ~1 frame of latency. -// Somewhere in between may hide screen tearing, or it may not. -void Pacer::setTearOffset(double offset) { - m_TearCtl.setTearOffset(offset); -} - -double Pacer::getTearOffset() { - return m_TearCtl.getTearOffset(); -} diff --git a/Streaming/Pacer.h b/Streaming/Pacer.h index 085d3dc8..ea85fbe7 100644 --- a/Streaming/Pacer.h +++ b/Streaming/Pacer.h @@ -4,7 +4,6 @@ #include #include #include "PacerRational.h" -#include "PacerTearController.h" #include "Utils.hpp" #include "VideoRenderer.h" @@ -22,9 +21,7 @@ class Pacer { void waitForFrame(); bool renderOnMainThread(std::shared_ptr &sceneRenderer); void waitUntilPresentTarget(); - void afterPresent(); - void setTearOffset(double offset); - double getTearOffset(); + void afterPresent(int64_t presentTimeQpc); void submitFrame(AVFrame *frame); private: @@ -67,6 +64,5 @@ class Pacer { std::mutex m_VsyncMutex; std::condition_variable m_VsyncSignalled; uint64_t m_VsyncSeq; - - TearController m_TearCtl; + double m_AvgDriftMs; }; diff --git a/Streaming/PacerTearController.h b/Streaming/PacerTearController.h deleted file mode 100644 index d2027da6..00000000 --- a/Streaming/PacerTearController.h +++ /dev/null @@ -1,100 +0,0 @@ -#pragma once -#include -#include -#include - -class EmaBiasCorrector { - public: - explicit EmaBiasCorrector(double halfLifeSamples = 60.0, - double targetEarlyMs = 0.05, - double maxPerFrameStepMs = 0.20, - double clampInputMs = 1.0) - : targetEarlyMs_(targetEarlyMs), - maxPerFrameStepMs_(std::max(0.0, maxPerFrameStepMs)), - clampInputMs_(std::max(0.0, clampInputMs)) { - setHalfLife(halfLifeSamples); - } - - void setHalfLife(double halfLifeSamples) { - if (halfLifeSamples <= 1.0) - alpha_ = 1.0; - else - alpha_ = 1.0 - std::pow(2.0, -1.0 / halfLifeSamples); - } - - void addSample(double driftMs) { - const double x = std::clamp(driftMs, -clampInputMs_, clampInputMs_); - if (!has_) { - ema_ = x; - has_ = true; - } else { - ema_ += alpha_ * (x - ema_); - } - const double raw = ema_ - targetEarlyMs_; - correctionMs_ = std::clamp(raw, -maxPerFrameStepMs_, +maxPerFrameStepMs_); - } - - double correctionMs() const { - return has_ ? correctionMs_ : 0.0; - } - double emaMs() const { - return has_ ? ema_ : 0.0; - } - - private: - double alpha_ = 0.1; - double ema_ = 0.0; - double correctionMs_ = 0.0; - double targetEarlyMs_ = 0.05; - double maxPerFrameStepMs_ = 0.25; - double clampInputMs_ = 2.0; - bool has_ = false; -}; - -class TearController { - public: - explicit TearController(double desiredTearFraction = 0.0, - double vblankPeriodMs = 0.0, - double safetyOffsetMs = 0.02) - : ready_(false), - desiredTearFraction_(desiredTearFraction), - vblankPeriodMs_(vblankPeriodMs), - safetyOffsetMs_(std::max(0.0, safetyOffsetMs)), - qpcPerMs_(UsToQpc(1000)) { - } - - void initializeTiming(double vblankPeriodMs, double safetyOffsetMs) { - vblankPeriodMs_ = vblankPeriodMs; - safetyOffsetMs_ = safetyOffsetMs; - ready_ = true; - } - - void setTearOffset(double f) { - desiredTearFraction_.store(std::clamp(f, 0.0, 1.0), std::memory_order_release); - } - - double getTearOffset() const { - return desiredTearFraction_.load(std::memory_order_acquire); - } - - int64_t targetPresentQpc(int64_t nextVblankQpc) const { - if (!ready_) { - throw std::runtime_error("TearController is not initialized"); - } - const double offsetMs = desiredTearFraction_.load(std::memory_order_acquire) * vblankPeriodMs_ + safetyOffsetMs_ + ema_.correctionMs(); - const double clamped = std::clamp(offsetMs, 0.0, std::max(0.0, vblankPeriodMs_ - 0.10)); // make sure we don't push into next vblank - return nextVblankQpc + (int64_t)std::llround(clamped * qpcPerMs_); - } - - void addDriftSample(double driftMs) { - ema_.addSample(driftMs); - } - - private: - bool ready_; - std::atomic desiredTearFraction_; - double vblankPeriodMs_; - double safetyOffsetMs_; - int64_t qpcPerMs_; - EmaBiasCorrector ema_; -}; diff --git a/Streaming/moonlight_xbox_dxMain.cpp b/Streaming/moonlight_xbox_dxMain.cpp index 6cb0a5d1..06ac0a1a 100644 --- a/Streaming/moonlight_xbox_dxMain.cpp +++ b/Streaming/moonlight_xbox_dxMain.cpp @@ -123,9 +123,10 @@ void moonlight_xbox_dxMain::StartRenderLoop() if (Render()) { Pacer::instance().waitUntilPresentTarget(); + int64_t presentTimeQpc = QpcNow(); m_deviceResources->Present(); - Pacer::instance().afterPresent(); + Pacer::instance().afterPresent(presentTimeQpc); int64_t afterPresent = QpcNow(); if (lastPresentTime > 0) { @@ -537,11 +538,3 @@ void moonlight_xbox_dxMain::SetImGui(bool isVisible) { Utils::Logf("ImGui graphs are now %d\n", isVisible ? 1 : 0); }); } - -void moonlight_xbox_dxMain::SetTearOffset(double offset) { - Pacer::instance().setTearOffset(offset); -} - -double moonlight_xbox_dxMain::GetTearOffset() { - return Pacer::instance().getTearOffset(); -} diff --git a/Streaming/moonlight_xbox_dxMain.h b/Streaming/moonlight_xbox_dxMain.h index 591d820c..abd9ce0b 100644 --- a/Streaming/moonlight_xbox_dxMain.h +++ b/Streaming/moonlight_xbox_dxMain.h @@ -39,8 +39,6 @@ namespace moonlight_xbox_dx bool ToggleLogs(); bool ToggleStats(); void SetImGui(bool isVisible); - void SetTearOffset(double offset); - double GetTearOffset(); private: void ProcessInput(); void Update(); From 1d8d253aaac5bb6551fc6861fcf4159ab7c58e74 Mon Sep 17 00:00:00 2001 From: Andy Grundman Date: Wed, 29 Oct 2025 14:51:45 -0400 Subject: [PATCH 22/22] Fix various CodeRabbit issues: moonlight-xbox-dx.vcxproj.filters: remove duplicate items Remove unused Microsoft code in DirectXHelper LogRenderer: more efficient warning message merge with dxhelper Streaming/PacerRational.h: cleanup intrinsic VideoRenderer: include libavutil headers HostSettingsPage: include cmath PlotDesc.h: check size of kPlotDescs array moonlight_xbox_dxMain Input: loop should run at 1000hz MoonlightClient.cpp: use std::lround when constructing config.clientRefreshRateX100 State/Stats.cpp: minor type fixes FFmpegDecoder: remove unused code Set ffmpeg log level to AV_LOG_ERROR Pacer: fix newline in log message Fix minor CR nitsquashs HostSettingsPage: fix HDR4KNote GridColumn HostSettingsPage: handle empty fps list MsToQpc: fix rounding FQLog: fix counter FFmpegDecoder: correctly unref hw_device_ctx FFmpegDecoder: free pkt correctly Pacer: fix m_Running to be atomic Pacer: pass QpcFreq to rational StatsRenderer: fix row2 FloatBuffer: return size_t instead of int vcpkg: remove ffmpeg features: avdevice, x264 --- Common/DeviceResources.cpp | 6 +- Common/DirectXHelper.cpp | 104 -- Common/DirectXHelper.h | 39 - Common/TextConsole.cpp | 2 +- Pages/HostSettingsPage.xaml | 2 +- Pages/HostSettingsPage.xaml.cpp | 2 + Plot/PlotDesc.h | 2 + State/MoonlightClient.cpp | 5 +- State/Stats.cpp | 17 +- Streaming/FFmpegDecoder.cpp | 15 +- Streaming/FFmpegDecoder.h | 2 - Streaming/LogRenderer.cpp | 7 +- Streaming/Pacer.cpp | 22 +- Streaming/Pacer.h | 8 +- Streaming/PacerRational.h | 10 +- Streaming/StatsRenderer.cpp | 8 +- Streaming/VideoRenderer.cpp | 8 +- Streaming/VideoRenderer.h | 5 +- Streaming/moonlight_xbox_dxMain.cpp | 4 +- Streaming/moonlight_xbox_dxMain.h | 4 - Utils/FloatBuffer.cpp | 4 +- Utils/FloatBuffer.h | 2 +- moonlight-xbox-dx.vcxproj | 1 - moonlight-xbox-dx.vcxproj.filters | 2562 +-------------------------- pch.h | 10 +- vcpkg.json | 4 +- 26 files changed, 71 insertions(+), 2784 deletions(-) delete mode 100644 Common/DirectXHelper.cpp diff --git a/Common/DeviceResources.cpp b/Common/DeviceResources.cpp index f843caf1..cfc777aa 100644 --- a/Common/DeviceResources.cpp +++ b/Common/DeviceResources.cpp @@ -668,15 +668,15 @@ void DX::DeviceResources::GetUWPPixelDimensions(uint32_t *width, uint32_t *heigh if (info.vendorId == GAMING_DEVICE_VENDOR_ID_MICROSOFT) { // Running on an Xbox auto mode = HdmiDisplayInformation::GetForCurrentView()->GetCurrentDisplayMode(); - *width = std::max((uint32_t)1920, mode->ResolutionWidthInRawPixels); + *width = std::max((uint32_t)1920, mode->ResolutionWidthInRawPixels); *height = std::max((uint32_t)1080, mode->ResolutionHeightInRawPixels); } else { // Running in Windows const LONG32 resolution_scale = static_cast(DisplayInformation::GetForCurrentView()->ResolutionScale); auto surface_scale = static_cast(resolution_scale) / 100.0f; - *width = (uint32_t)CoreWindow::GetForCurrentThread()->Bounds.Width * surface_scale; - *height = (uint32_t)CoreWindow::GetForCurrentThread()->Bounds.Height * surface_scale; + *width = static_cast(CoreWindow::GetForCurrentThread()->Bounds.Width * surface_scale); + *height = static_cast(CoreWindow::GetForCurrentThread()->Bounds.Height * surface_scale); } } diff --git a/Common/DirectXHelper.cpp b/Common/DirectXHelper.cpp deleted file mode 100644 index 12e11fca..00000000 --- a/Common/DirectXHelper.cpp +++ /dev/null @@ -1,104 +0,0 @@ -#include "pch.h" -#include "Common\DirectXHelper.h" -#include "Common\DeviceResources.h" -#include - -using namespace Microsoft::WRL; - -DX::GpuPerformanceTimer::GpuPerformanceTimer( - const std::shared_ptr& deviceResources) : - m_deviceResources(deviceResources) -{ - ID3D11Device* device = m_deviceResources->GetD3DDevice(); - - const D3D11_QUERY_DESC timestampQuery = CD3D11_QUERY_DESC(D3D11_QUERY_TIMESTAMP); - const D3D11_QUERY_DESC disjointQuery = CD3D11_QUERY_DESC(D3D11_QUERY_TIMESTAMP_DISJOINT); - - for (uint32_t i = 0; i < QueryCount; ++i) - { - device->CreateQuery(×tampQuery, m_startTimestampQuery[i].ReleaseAndGetAddressOf()); - device->CreateQuery(×tampQuery, m_endTimestampQuery[i].ReleaseAndGetAddressOf()); - device->CreateQuery(&disjointQuery, m_disjointQuery[i].ReleaseAndGetAddressOf()); - } -} - -void DX::GpuPerformanceTimer::StartTimerForFrame() -{ - ID3D11DeviceContext* context = m_deviceResources->GetD3DDeviceContext(); - - // Just some book-keeping to see how long this algorithm is running - m_currentQuery = (m_currentQuery + 1) % QueryCount; - const uint32_t ReadQuery = (m_currentQuery + 1) % QueryCount; - - if (m_currentFrameIndex >= QueryCount) - { - D3D11_QUERY_DATA_TIMESTAMP_DISJOINT disjointData; - while (context->GetData(m_disjointQuery[ReadQuery].Get(), &disjointData, sizeof(disjointData), 0) != S_OK); - - if (disjointData.Disjoint == 0) - { - uint64_t start; - while (context->GetData(m_startTimestampQuery[ReadQuery].Get(), &start, sizeof(start), 0) != S_OK); - - uint64_t end; - while (context->GetData(m_endTimestampQuery[ReadQuery].Get(), &end, sizeof(end), 0) != S_OK); - - constexpr float secondsToMilliseconds = 1000.0f; - const float ticksToSeconds = 1.0f / static_cast(disjointData.Frequency); - const uint64_t ticksElapsed = end - start; - - m_processingTimeMs = static_cast(ticksElapsed) * ticksToSeconds * secondsToMilliseconds; - - // Reset every ~10 seconds so the min/max has a chance to update - if (m_currentFrameIndex % 300 == 0) - { - m_processingTimeMinMs = +std::numeric_limits::infinity(); - m_processingTimeMaxMs = -std::numeric_limits::infinity(); - } - - m_processingTimeMinMs = std::min(m_processingTimeMinMs, m_processingTimeMs); - m_processingTimeMaxMs = std::max(m_processingTimeMaxMs, m_processingTimeMs); - } - else - { - m_processingTimeMs = std::numeric_limits::quiet_NaN();; - } - - m_processingTimeHistory[m_processingTimeHistoryIndex] = m_processingTimeMs; - m_processingTimeHistoryIndex = (m_processingTimeHistoryIndex + 1) % TimeHistoryCount; - - m_processingTimeAvgMs = std::accumulate(m_processingTimeHistory, m_processingTimeHistory + TimeHistoryCount, 0.0f) / static_cast(TimeHistoryCount); - } - - ++m_currentFrameIndex; - - context->Begin(m_disjointQuery[m_currentQuery].Get()); - context->End(m_startTimestampQuery[m_currentQuery].Get()); -} - -void DX::GpuPerformanceTimer::EndTimerForFrame() -{ - ID3D11DeviceContext* context = m_deviceResources->GetD3DDeviceContext(); - context->End(m_endTimestampQuery[m_currentQuery].Get()); - context->End(m_disjointQuery[m_currentQuery].Get()); -} - -float DX::GpuPerformanceTimer::GetFrameTime() const -{ - return m_processingTimeMs; -} - -float DX::GpuPerformanceTimer::GetAvgFrameTime() const -{ - return m_processingTimeAvgMs; -} - -float DX::GpuPerformanceTimer::GetMinFrameTime() const -{ - return m_processingTimeMinMs; -} - -float DX::GpuPerformanceTimer::GetMaxFrameTime() const -{ - return m_processingTimeMaxMs; -} diff --git a/Common/DirectXHelper.h b/Common/DirectXHelper.h index 93c0460e..50d8e8fc 100644 --- a/Common/DirectXHelper.h +++ b/Common/DirectXHelper.h @@ -79,43 +79,4 @@ namespace DX } #endif - class DeviceResources; - - // Used to track time spent executing Gpu work - class GpuPerformanceTimer - { - public: - GpuPerformanceTimer( - const std::shared_ptr& deviceResources); - - void StartTimerForFrame(); - void EndTimerForFrame(); - - float GetFrameTime() const; - float GetAvgFrameTime() const; - float GetMinFrameTime() const; - float GetMaxFrameTime() const; - - private: - std::shared_ptr m_deviceResources; - - static constexpr uint32_t QueryCount = 5u; - - Microsoft::WRL::ComPtr m_startTimestampQuery[QueryCount]; - Microsoft::WRL::ComPtr m_endTimestampQuery[QueryCount]; - Microsoft::WRL::ComPtr m_disjointQuery[QueryCount]; - - uint32_t m_currentQuery = 0u; - uint32_t m_currentFrameIndex = 0u; - - static constexpr uint32_t TimeHistoryCount = 64u; - - uint32_t m_processingTimeHistoryIndex = 0u; - float m_processingTimeHistory[TimeHistoryCount] = {}; - - float m_processingTimeMs = 0.0f; - float m_processingTimeAvgMs = 0.0f; - float m_processingTimeMinMs = +std::numeric_limits::infinity(); - float m_processingTimeMaxMs = -std::numeric_limits::infinity(); - }; } diff --git a/Common/TextConsole.cpp b/Common/TextConsole.cpp index 780d1c74..6f842999 100644 --- a/Common/TextConsole.cpp +++ b/Common/TextConsole.cpp @@ -201,7 +201,7 @@ void TextConsole::SetWindow(const RECT& layout) std::swap(buffer, m_buffer); std::swap(lines, m_lines); - Utils::Logf("TextConsole initialized at (%d,%d) - (%d,%d) with %d rows, %d columns\n", + Utils::Logf("TextConsole initialized at (%d,%d) - (%d,%d) with %u rows, %u columns\n", layout.left, layout.top, layout.right, layout.bottom, m_rows, m_columns); if ((m_currentColumn >= m_columns) || (m_currentLine >= m_rows)) diff --git a/Pages/HostSettingsPage.xaml b/Pages/HostSettingsPage.xaml index a71f90e8..4db0cc81 100644 --- a/Pages/HostSettingsPage.xaml +++ b/Pages/HostSettingsPage.xaml @@ -71,7 +71,7 @@ Name="EnableHDRCheckbox" Grid.Row="7" Grid.Column="1" IsChecked="{x:Bind Host.EnableHDR,Mode=TwoWay}" /> + Name="HDR4KNote" Grid.Row="7" Grid.Column="1" Visibility="Collapsed"> HDR requires Xbox system resolution set to 4K. diff --git a/Pages/HostSettingsPage.xaml.cpp b/Pages/HostSettingsPage.xaml.cpp index 69e70ba5..77399e5c 100644 --- a/Pages/HostSettingsPage.xaml.cpp +++ b/Pages/HostSettingsPage.xaml.cpp @@ -8,6 +8,7 @@ #include "MoonlightSettings.xaml.h" #include "Utils.hpp" #include +#include // sqrtf, lround using namespace Windows::UI::Core; using namespace moonlight_xbox_dx; @@ -132,6 +133,7 @@ void HostSettingsPage::ResolutionSelector_SelectionChanged(Platform::Object^ sen void HostSettingsPage::FPSSelector_SelectionChanged(Platform::Object^ sender, Windows::UI::Xaml::Controls::SelectionChangedEventArgs^ e) { + if (e->AddedItems->Size == 0) return; int selectedFPS = (int)e->AddedItems->GetAt(0); // Default to a new bitrate if a new FPS was chosen diff --git a/Plot/PlotDesc.h b/Plot/PlotDesc.h index e52b1f69..ee47403d 100644 --- a/Plot/PlotDesc.h +++ b/Plot/PlotDesc.h @@ -43,3 +43,5 @@ inline constexpr std::array kPlotDescs = {{ {"Graph overhead", PLOT_LABEL_MIN_MAX_AVG, "ms", -0.1f, 6.0f, 0.0f, 0.0f}, {"Etc...", PLOT_LABEL_MIN_MAX_AVG, "ms", -0.1f, 50.0f, 0.0f, 49.9f}, }}; + +static_assert(kPlotDescs.size() == PlotCount, "Plot descriptors out of sync with PlotType enum"); diff --git a/State/MoonlightClient.cpp b/State/MoonlightClient.cpp index 66f1e73b..1c11c221 100644 --- a/State/MoonlightClient.cpp +++ b/State/MoonlightClient.cpp @@ -12,6 +12,7 @@ extern "C" { #include #include #include +#include using namespace moonlight_xbox_dx; using namespace Windows::Gaming::Input; @@ -224,7 +225,7 @@ int MoonlightClient::StartStreaming(std::shared_ptr res, St double rr = res->GetRefreshRate(); switch (config.fps) { case 120: - config.clientRefreshRateX100 = (int)(rr * 100.0); + config.clientRefreshRateX100 = (int)std::lround(rr * 100.0); break; case 60: if (rr >= 120.0) { @@ -232,7 +233,7 @@ int MoonlightClient::StartStreaming(std::shared_ptr res, St } else if (rr >= 119.0) { config.clientRefreshRateX100 = 5994; } else { - config.clientRefreshRateX100 = (int)(rr * 100.0); + config.clientRefreshRateX100 = (int)std::lround(rr * 100.0); } break; default: diff --git a/State/Stats.cpp b/State/Stats.cpp index a924e3f7..d915356d 100644 --- a/State/Stats.cpp +++ b/State/Stats.cpp @@ -93,10 +93,11 @@ void Stats::SubmitVideoBytesAndReassemblyTime(uint32_t length, PDECODE_UNIT deco // Host frametime graph, uses raw 90kHz units to avoid rounding errors static uint32_t lastHostPts = 0; - if (lastHostPts > 0) { - ImGuiPlots::instance().observeFloat(PLOT_HOST_FRAMETIME, (float)((decodeUnit->rtpTimestamp - lastHostPts) / 90.0f)); + if (lastHostPts != 0) { + const uint32_t delta90k = (uint32_t)(decodeUnit->rtpTimestamp - lastHostPts); // wrap-safe + ImGuiPlots::instance().observeFloat(PLOT_HOST_FRAMETIME, (float)(delta90k / 90.0f)); } - lastHostPts = decodeUnit->rtpTimestamp; + lastHostPts = (uint32_t)decodeUnit->rtpTimestamp; } // Time in milliseconds we spent decoding one frame, it is added up to later be divided by decodedFrames @@ -266,7 +267,7 @@ void Stats::formatVideoStats(DX::StepTimer const& timer, VIDEO_STATS& stats, cha ffmpeg.height, stats.totalFps, codecString); - if (ret < 0 || ret >= length - offset) { + if (ret < 0 || (size_t)ret >= (length - offset)) { Utils::Log("Error: stringifyVideoStats length overflow\n"); return; } @@ -288,7 +289,7 @@ void Stats::formatVideoStats(DX::StepTimer const& timer, VIDEO_STATS& stats, cha stats.receivedFps, stats.decodedFps, stats.renderedFps); - if (ret < 0 || ret >= length - offset) { + if (ret < 0 || (size_t)ret >= (length - offset)) { Utils::Log("Error: stringifyVideoStats length overflow\n"); return; } @@ -303,7 +304,7 @@ void Stats::formatVideoStats(DX::StepTimer const& timer, VIDEO_STATS& stats, cha (double)stats.minHostProcessingLatency / 10, (double)stats.maxHostProcessingLatency / 10, (double)stats.totalHostProcessingLatency / 10 / stats.framesWithHostProcessingLatency); - if (ret < 0 || ret >= length - offset) { + if (ret < 0 || (size_t)ret >= (length - offset)) { Utils::Log("Error: stringifyVideoStats length overflow\n"); return; } @@ -315,7 +316,7 @@ void Stats::formatVideoStats(DX::StepTimer const& timer, VIDEO_STATS& stats, cha ret = snprintf(&output[offset], length - offset, "Host processing latency min/max/avg: -/-/- ms\n"); - if (ret < 0 || ret >= length - offset) { + if (ret < 0 || (size_t)ret >= (length - offset)) { Utils::Log("Error: stringifyVideoStats length overflow\n"); return; } @@ -349,7 +350,7 @@ void Stats::formatVideoStats(DX::StepTimer const& timer, VIDEO_STATS& stats, cha stats.renderedFrames ? (double)stats.totalPacerTimeUs / 1000.0 / stats.renderedFrames : 0.0f, stats.renderedFrames ? (double)stats.totalPresentDisplayMs / stats.renderedFrames : 0.0f, stats.renderedFrames ? (double)stats.totalRenderTimeUs / 1000.0 / stats.renderedFrames : 0.0f); - if (ret < 0 || ret >= length - offset) { + if (ret < 0 || (size_t)ret >= (length - offset)) { Utils::Log("Error: stringifyVideoStats length overflow\n"); return; } diff --git a/Streaming/FFmpegDecoder.cpp b/Streaming/FFmpegDecoder.cpp index 75bebeb9..166067ca 100644 --- a/Streaming/FFmpegDecoder.cpp +++ b/Streaming/FFmpegDecoder.cpp @@ -134,6 +134,7 @@ namespace moonlight_xbox_dx { } decoder_ctx->hw_device_ctx = av_buffer_ref(hw_device_ctx); + av_buffer_unref(&hw_device_ctx); decoder_ctx->pix_fmt = AV_PIX_FMT_D3D11; decoder_ctx->sw_pix_fmt = (videoFormat & VIDEO_FORMAT_MASK_10BIT) ? AV_PIX_FMT_P010 : AV_PIX_FMT_NV12; decoder_ctx->pkt_timebase.num = 1; @@ -166,17 +167,6 @@ namespace moonlight_xbox_dx { pMultithread->SetMultithreadProtected(TRUE); pMultithread->Release(); } - - GAMING_DEVICE_MODEL_INFORMATION info = {}; - GetGamingDeviceModelInformation(&info); - if (info.vendorId == GAMING_DEVICE_VENDOR_ID_MICROSOFT && - (info.deviceId == GAMING_DEVICE_DEVICE_ID_XBOX_ONE || - info.deviceId == GAMING_DEVICE_DEVICE_ID_XBOX_ONE_S || - info.deviceId == GAMING_DEVICE_DEVICE_ID_XBOX_ONE_X || - info.deviceId == GAMING_DEVICE_DEVICE_ID_XBOX_ONE_X_DEVKIT)) { - Utils::Log("Using hack for Xbox One Consoles"); - hackWait = true; - } return 0; } @@ -251,6 +241,7 @@ namespace moonlight_xbox_dx { int err = avcodec_send_packet(decoder_ctx, pkt); av_packet_unref(pkt); + av_packet_free(&pkt); if (err < 0) { char ffmpegError[1024]; av_strerror(err, ffmpegError, 1024); @@ -297,7 +288,7 @@ namespace moonlight_xbox_dx { // Restore default log level after a successful decode if (av_log_get_level() > 0) { - av_log_set_level(AV_LOG_QUIET); + av_log_set_level(AV_LOG_ERROR); } return DR_OK; diff --git a/Streaming/FFmpegDecoder.h b/Streaming/FFmpegDecoder.h index bbed81ab..12d98f52 100644 --- a/Streaming/FFmpegDecoder.h +++ b/Streaming/FFmpegDecoder.h @@ -36,8 +36,6 @@ class FFMpegDecoder { int SubmitDecodeUnit(PDECODE_UNIT decodeUnit); static FFMpegDecoder *getInstance(); static DECODER_RENDERER_CALLBACKS getDecoder(); - std::recursive_mutex mutex; - bool hackWait = false; int videoFormat, width, height; private: diff --git a/Streaming/LogRenderer.cpp b/Streaming/LogRenderer.cpp index 62040e5b..a9f10050 100644 --- a/Streaming/LogRenderer.cpp +++ b/Streaming/LogRenderer.cpp @@ -32,7 +32,6 @@ void LogRenderer::Update(DX::StepTimer const& timer) static double lastUpdateSeconds = 0.0; if (m_visible && timer.GetTotalSeconds() - lastUpdateSeconds >= 1.0) { m_console->Clear(); - m_warningConsole->Clear(); Utils::logMutex.lock(); std::vector lines = Utils::GetLogLines(); @@ -41,8 +40,6 @@ void LogRenderer::Update(DX::StepTimer const& timer) } Utils::logMutex.unlock(); - m_warningConsole->Write(L"Warning: Viewing logs reduces performance"); - lastUpdateSeconds = timer.GetTotalSeconds(); } } @@ -95,4 +92,8 @@ void LogRenderer::ReleaseDeviceDependentResources() void LogRenderer::ToggleVisible() { std::lock_guard lock(m_mutex); m_visible = m_visible == true ? false : true; + if (m_visible) { + m_warningConsole->Clear(); + m_warningConsole->Write(L"Warning: Viewing logs reduces performance"); + } } diff --git a/Streaming/Pacer.cpp b/Streaming/Pacer.cpp index 0a3ba68f..5bdd4826 100644 --- a/Streaming/Pacer.cpp +++ b/Streaming/Pacer.cpp @@ -91,7 +91,7 @@ Pacer::Pacer() } void Pacer::deinit() { - m_Running = false; + m_Running.store(false, std::memory_order_release); m_Stopping.store(true, std::memory_order_release); m_RenderQueueNotEmpty.notify_all(); @@ -136,7 +136,7 @@ void Pacer::init(const std::shared_ptr &res, int streamFps, m_StreamFps = streamFps; m_RefreshRate = refreshRate; - Utils::Logf("Pacer: init target %.2f Hz with %d FPS stream", + Utils::Logf("Pacer: init target %.2f Hz with %d FPS stream\n", m_RefreshRate, m_StreamFps); // If running on Xbox at 120hz, emulate the vsync interval @@ -157,7 +157,7 @@ void Pacer::init(const std::shared_ptr &res, int streamFps, m_BackPacerThread = std::thread(&Pacer::backPacer, this); } - m_Running = true; + m_Running.store(true, std::memory_order_release); } // Vsync Emulator @@ -208,7 +208,7 @@ void Pacer::vsyncEmulator() { } int64_t vsyncQpc = waitForVBlank(); - m_VsyncPeriod.initFromHz(hzNum, hzDen, vsyncQpc); + m_VsyncPeriod.initFromHz(hzNum, hzDen, vsyncQpc, QpcFreq()); Utils::Logf("Pacer: using emulated integer vsync timer %lu/%lu based on system refresh rate %.2f Hz\n", hzNum, hzDen, m_RefreshRate); @@ -232,7 +232,7 @@ void Pacer::vsyncEmulator() { // Align to a real vblank edge baseQpc = waitForVBlank(); - m_VsyncPeriod.initFromHz(hzNum, hzDen, baseQpc); + m_VsyncPeriod.initFromHz(hzNum, hzDen, baseQpc, QpcFreq()); FQLog("resync to hardware vsync, waited %.3fms\n", QpcToMs(baseQpc - pre)); lastVblankSeq = m_VsyncSeq; @@ -315,7 +315,7 @@ void Pacer::vsyncHardware() { // no tearing to worry about, we will just use it for timing int64_t vsyncQpc = waitForVBlank(); - m_VsyncPeriod.initFromHz(hzNum, hzDen, vsyncQpc); + m_VsyncPeriod.initFromHz(hzNum, hzDen, vsyncQpc, QpcFreq()); Utils::Logf("Pacer: using integer vsync timer %lu/%lu based on system refresh rate %.2f Hz\n", hzNum, hzDen, m_RefreshRate); @@ -332,7 +332,7 @@ void Pacer::vsyncHardware() { continue; } - m_VsyncPeriod.initFromHz(hzNum, hzDen, now); + m_VsyncPeriod.initFromHz(hzNum, hzDen, now, QpcFreq()); // Compute next period m_VsyncPeriod.step(); @@ -506,7 +506,7 @@ void Pacer::enqueueFrameForRenderingAndUnlock(AVFrame *frame) { // Main render thread void Pacer::waitForFrame() { - if (!m_Running) return; + if (!running()) return; // Wait for the renderer to be ready for the next frame #ifdef FRAME_QUEUE_VERBOSE @@ -528,7 +528,7 @@ void Pacer::waitForFrame() { // called by render thread bool Pacer::renderOnMainThread(std::shared_ptr &sceneRenderer) { - if (!m_Running) return false; + if (!running()) return false; AVFrame *frame = nullptr; @@ -620,7 +620,7 @@ void Pacer::frontPacer(std::shared_ptr &sceneRenderer, AVFrame *f // called by render thread void Pacer::waitUntilPresentTarget() { - if (!m_Running) return; + if (!running()) return; FQLog("waitUntilPresentTarget(): waiting for %.3fms\n", QpcToMs(m_PresentTargetQpc - QpcNow())); @@ -629,7 +629,7 @@ void Pacer::waitUntilPresentTarget() { // called by render thread void Pacer::afterPresent(int64_t presentTimeQpc) { - if (!m_Running) return; + if (!running()) return; // Called immediately after Present(), check our timings and adjust future timings const int64_t driftQpc = presentTimeQpc - m_PresentVsyncQpc; diff --git a/Streaming/Pacer.h b/Streaming/Pacer.h index ea85fbe7..e1925e43 100644 --- a/Streaming/Pacer.h +++ b/Streaming/Pacer.h @@ -29,10 +29,14 @@ class Pacer { Pacer(const Pacer &) = delete; Pacer &operator=(const Pacer &) = delete; - bool stopping() const noexcept { + inline bool stopping() const noexcept { return m_Stopping.load(std::memory_order_acquire); } + inline bool running() const noexcept { + return m_Running.load(std::memory_order_acquire); + } + void vsyncEmulator(); void vsyncHardware(); void backPacer(); @@ -42,7 +46,6 @@ class Pacer { void dropFrameForEnqueue(std::deque &queue); int64_t waitForVBlank(); - bool m_Running; std::shared_ptr m_DeviceResources; std::thread m_VsyncThread; std::thread m_BackPacerThread; @@ -53,6 +56,7 @@ class Pacer { std::mutex m_FrameQueueLock; std::condition_variable m_RenderQueueNotEmpty; std::condition_variable m_PacingQueueNotEmpty; + std::atomic m_Running{false}; std::atomic m_Stopping{false}; int m_StreamFps; double m_RefreshRate; diff --git a/Streaming/PacerRational.h b/Streaming/PacerRational.h index ed6a4507..6ab43145 100644 --- a/Streaming/PacerRational.h +++ b/Streaming/PacerRational.h @@ -3,10 +3,10 @@ #pragma once #include -#include #include - +#if defined(_MSC_VER) #pragma intrinsic(_umul128, _udiv128) +#endif // Represents a period in QPC ticks using an integer part plus a fractional accumulator. // This avoids drift at fractional Hz such as 59.94 or 119.88. @@ -19,11 +19,9 @@ struct QpcRationalPeriod { // Initialize from exact Hz as a rational number: Hz = num/den. // Period(QPC) = QpcFreq * den / num. - void initFromHz(uint32_t hzNum, uint32_t hzDen, int64_t baseQpc) { - const uint64_t f = (uint64_t)QpcFreq(); - + void initFromHz(uint32_t hzNum, uint32_t hzDen, int64_t baseQpc, int64_t qpcFreq) { uint64_t hi = 0; - uint64_t lo = _umul128(f, (uint64_t)hzDen, &hi); + uint64_t lo = _umul128((uint64_t)qpcFreq, (uint64_t)hzDen, &hi); uint64_t r = 0; uint64_t q = _udiv128(hi, lo, hzNum, &r); diff --git a/Streaming/StatsRenderer.cpp b/Streaming/StatsRenderer.cpp index 9de91543..28f5867e 100644 --- a/Streaming/StatsRenderer.cpp +++ b/Streaming/StatsRenderer.cpp @@ -77,8 +77,6 @@ void StatsRenderer::RenderGraphs() { (float *)malloc(sizeof(float) * 512), (float *)malloc(sizeof(float) * 512)}; - ImGuiIO &io = ImGui::GetIO(); - float graphW = 850.0f * (m_displayWidth / 3840.0f); float graphH = 120.0f * (m_displayHeight / 2160.0f); float opacity = 0.8f; @@ -110,7 +108,7 @@ void StatsRenderer::RenderGraphs() { float minY = 0.0f; float maxY = 0.0f; - int countF = plot.buffer.copyInto(buffers[i], 512, minY, maxY); + std::size_t countF = plot.buffer.copyInto(buffers[i], 512, minY, maxY); float avgF = plot.buffer.average(); if (!countF) { return; @@ -161,8 +159,8 @@ void StatsRenderer::RenderGraphs() { } ImGui::Dummy(ImVec2(1.0f, itemSpacingY)); - const int row2[4] = {PLOT_HOST_FRAMETIME, PLOT_PRESENT_PACING, PLOT_DROPPED_PACER}; - for (int c = 0; c < 4; ++c) { + const int row2[3] = {PLOT_HOST_FRAMETIME, PLOT_PRESENT_PACING, PLOT_DROPPED_PACER}; + for (int c = 0; c < 3; ++c) { if (c > 0) ImGui::SameLine(0.0f, itemSpacingX); draw_plot(row2[c], graphW, graphH); } diff --git a/Streaming/VideoRenderer.cpp b/Streaming/VideoRenderer.cpp index bc777bd5..fd177c57 100644 --- a/Streaming/VideoRenderer.cpp +++ b/Streaming/VideoRenderer.cpp @@ -103,7 +103,7 @@ void VideoRenderer::Update(DX::StepTimer const& timer) } -void VideoRenderer::EnsureYuvTargets(ID3D11Device* dev, DXGI_FORMAT fmt, UINT w, UINT h) { +void VideoRenderer::ensureYuvTargets(ID3D11Device* dev, DXGI_FORMAT fmt, UINT w, UINT h) { if (m_yuvTexture && m_yuvFormat == fmt && m_yuvWidth == w && m_yuvHeight == h) return; @@ -157,7 +157,7 @@ bool VideoRenderer::Render(AVFrame *frame) { ffmpegTexture->GetDesc(&ffmpegDesc); int index = (int)(frame->data[1]); - EnsureYuvTargets(dev, ffmpegDesc.Format, frame->width, frame->height); + ensureYuvTargets(dev, ffmpegDesc.Format, frame->width, frame->height); ID3D11ShaderResourceView *nullSrvs[2] = {nullptr, nullptr}; ctx->PSSetShaderResources(0, 2, nullSrvs); @@ -539,8 +539,6 @@ void VideoRenderer::bindColorConversion(AVFrame* frame) void VideoRenderer::SetHDR(bool enabled) { - FFMpegDecoder::instance().mutex.lock(); - if (enabled) { SS_HDR_METADATA sunshineHdrMetadata; @@ -556,8 +554,6 @@ void VideoRenderer::SetHDR(bool enabled) // toggle the display to the correct state client->SetDisplayHDR(false, SS_HDR_METADATA{}); } - - FFMpegDecoder::instance().mutex.unlock(); } void VideoRenderer::Stop() { diff --git a/Streaming/VideoRenderer.h b/Streaming/VideoRenderer.h index a19856e9..434ed0aa 100644 --- a/Streaming/VideoRenderer.h +++ b/Streaming/VideoRenderer.h @@ -7,6 +7,8 @@ extern "C" { #include +#include // AVFrame +#include // AVColorTransferCharacteristic } namespace moonlight_xbox_dx @@ -30,7 +32,6 @@ namespace moonlight_xbox_dx void Update(DX::StepTimer const& timer); void scaleSourceToDestinationSurface(IRECT* src, IRECT* dst); void screenSpaceToNormalizedDeviceCoords(IRECT* src, FRECT* dst, int viewportWidth, int viewportHeight); - void EnsureYuvTargets(ID3D11Device* dev, DXGI_FORMAT fmt, UINT w, UINT h); bool Render(AVFrame* frame); void bindColorConversion(AVFrame* frame); void SetHDR(bool enabled); @@ -39,6 +40,8 @@ namespace moonlight_xbox_dx private: + void ensureYuvTargets(ID3D11Device* dev, DXGI_FORMAT fmt, UINT w, UINT h); + // Cached pointer to device resources. std::shared_ptr m_deviceResources; diff --git a/Streaming/moonlight_xbox_dxMain.cpp b/Streaming/moonlight_xbox_dxMain.cpp index 06ac0a1a..828b88d1 100644 --- a/Streaming/moonlight_xbox_dxMain.cpp +++ b/Streaming/moonlight_xbox_dxMain.cpp @@ -142,9 +142,9 @@ void moonlight_xbox_dxMain::StartRenderLoop() } auto inputItemHandler = ref new WorkItemHandler([this](IAsyncAction^ action) { - // Target period = 2000hz + // Target period = 1000hz using clock = std::chrono::high_resolution_clock; - const auto period = std::chrono::microseconds(2000); + const auto period = std::chrono::microseconds(1000); auto next = clock::now() + period; // Calculate the updated frame and render once per vertical blanking interval. diff --git a/Streaming/moonlight_xbox_dxMain.h b/Streaming/moonlight_xbox_dxMain.h index abd9ce0b..cfb60a26 100644 --- a/Streaming/moonlight_xbox_dxMain.h +++ b/Streaming/moonlight_xbox_dxMain.h @@ -6,10 +6,6 @@ #include "Streaming\StatsRenderer.h" #include "Pages\StreamPage.xaml.h" -namespace DX { - class GpuPerformanceTimer; -} - // Renders Direct2D and 3D content on the screen. namespace moonlight_xbox_dx { diff --git a/Utils/FloatBuffer.cpp b/Utils/FloatBuffer.cpp index b612edc0..b5c6239b 100644 --- a/Utils/FloatBuffer.cpp +++ b/Utils/FloatBuffer.cpp @@ -53,7 +53,7 @@ void FloatBuffer::push(float value) noexcept } } -int FloatBuffer::copyInto(float *outBuffer, std::size_t outSize, float &out_min, float &out_max) const +std::size_t FloatBuffer::copyInto(float *outBuffer, std::size_t outSize, float &out_min, float &out_max) const { std::lock_guard lock(mtx_); @@ -79,7 +79,7 @@ int FloatBuffer::copyInto(float *outBuffer, std::size_t outSize, float &out_min, out_min = min_; out_max = max_; - return outLen; + return static_cast(outLen); } void FloatBuffer::clear() noexcept diff --git a/Utils/FloatBuffer.h b/Utils/FloatBuffer.h index 3c2a5cf2..2af290a7 100644 --- a/Utils/FloatBuffer.h +++ b/Utils/FloatBuffer.h @@ -12,7 +12,7 @@ class FloatBuffer explicit FloatBuffer(std::size_t capacity = kDefaultCapacity); void push(float value) noexcept; - int copyInto(float *outBuffer, std::size_t outSize, float &out_min, float &out_max) const; + std::size_t copyInto(float *outBuffer, std::size_t outSize, float &out_min, float &out_max) const; void clear() noexcept; std::size_t capacity() const noexcept diff --git a/moonlight-xbox-dx.vcxproj b/moonlight-xbox-dx.vcxproj index 19ad4cf6..0743f95f 100644 --- a/moonlight-xbox-dx.vcxproj +++ b/moonlight-xbox-dx.vcxproj @@ -383,7 +383,6 @@ App.xaml - diff --git a/moonlight-xbox-dx.vcxproj.filters b/moonlight-xbox-dx.vcxproj.filters index b0d5eaa3..a65f7d7f 100644 --- a/moonlight-xbox-dx.vcxproj.filters +++ b/moonlight-xbox-dx.vcxproj.filters @@ -383,9 +383,6 @@ Source Files - - Source Files - @@ -641,2563 +638,6 @@ DLLs - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - DLLs - - - - DLLs - - - DLLs - @@ -3246,4 +686,4 @@ - \ No newline at end of file + diff --git a/pch.h b/pch.h index 2dbd21a3..655528a8 100644 --- a/pch.h +++ b/pch.h @@ -84,7 +84,8 @@ static inline double QpcToMs(int64_t qpc) { } static inline int64_t MsToQpc(double ms) { - const int64_t us = static_cast(ms * 1000.0 + 0.5); + const double us_d = ms * 1000.0; + const int64_t us = static_cast(us_d >= 0.0 ? us_d + 0.5 : us_d - 0.5); return UsToQpc(us); } @@ -103,13 +104,14 @@ static inline int64_t MsToQpc(double ms) { // Note: When FQLog is enabled, the spam is intense, so it only logs data for a short time #if !defined(NDEBUG) //#define FRAME_QUEUE_VERBOSE -static int __once = 0; #endif #ifdef FRAME_QUEUE_VERBOSE + #include + static std::atomic g_fqlog_counter{0}; #define FQLog(fmt, ...) \ - if (++__once > 200 && __once < 1000) \ - moonlight_xbox_dx::Utils::Logf("[%d] " fmt, ::GetCurrentThreadId(), ##__VA_ARGS__) + if (++g_fqlog_counter > 200 && g_fqlog_counter < 1000) \ + moonlight_xbox_dx::Utils::Logf("[%lu] " fmt, ::GetCurrentThreadId(), ##__VA_ARGS__) #else #if defined(_MSC_VER) #define FQLog(...) __noop diff --git a/vcpkg.json b/vcpkg.json index a9c55497..6d5e09d0 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -7,9 +7,7 @@ "default-features": false, "features": [ "avcodec", - "avdevice", - "swscale", - "x264" + "swscale" ] }, "curl",