Scope setts to audio DTS and verify A/V sync in the repair gate

HISTORY.md

@@ -11,8 +11,8 @@ Utility to optimize media files for Direct Play in Plex, Emby, Jellyfin, etc.
     - Verify now classifies the decode diagnostics deterministically as clean, a timestamp-only failure, or a decode error; a timestamp-only failure is repaired losslessly when the break is demux-visible and otherwise stays reported, and everything else fails (fail-closed, so an unrecognized diagnostic fails as a decode error).
     - The classification streams the output line by line, so memory stays bounded even when a file emits a warning per packet ([#827](https://github.com/ptr727/PlexCleaner/issues/827)).
   - Added a lossless timestamp repair as the first repair tier.
-    - When verification detects a demux-visible non-monotonic DTS, the audio packet timestamps are rewritten to be strictly monotonic using the `setts` bitstream filter with a stream copy (no re-encode), then re-verified.
-    - A regression gate compares the per-stream coded payload hash before and after and discards the result unless every stream is byte-identical, so the repair can never alter the media. The full re-encode repair remains for genuine decode corruption.
+    - When verification detects a demux-visible non-monotonic DTS on an audio stream, the audio packet timestamps are rewritten to be strictly monotonic using the `setts` bitstream filter with a stream copy (no re-encode), then re-verified. A video-stream DTS is not audio-repairable (a video `setts` would reorder B-frames) and stays reported.
+    - A regression gate compares the per-stream coded payload hash and the per-stream start and duration before and after, discarding the result unless every stream is byte-identical and no stream shifted beyond the A/V-sync tolerance, so the repair can neither alter the media nor drift the audio out of sync. The full re-encode repair remains for genuine decode corruption.
   - Consolidated the bitrate and DTS packet analyses into a single `ffprobe -show_packets` pass, computing the per-second bitrate and the per-stream DTS monotonicity together instead of reading packets twice.
   - Switched closed caption detection to `ffprobe -analyze_frames -show_entries stream=closed_captions`, replacing the `movie=...[out0+subcc]` lavfi filter and its QuickScan snippet-remux workaround; QuickScan now bounds the scan with `-read_intervals`.
   - Added the `DtsTimestampRepair` example plugin.

PlexCleaner/DtsInfo.cs

@@ -8,27 +8,46 @@ public sealed class DtsInfo
     // Count of non-monotonic packets per stream index
     private readonly Dictionary<long, int> _nonMonotonicByStream = [];
 
+    // Codec type per stream index, used to decide whether the audio setts filter can repair the DTS
+    private readonly Dictionary<long, string> _codecTypeByStream = [];
+
     // Stream indexes carrying a non-monotonic DTS, with the per-stream count
     public IReadOnlyDictionary<long, int> NonMonotonicByStream => _nonMonotonicByStream;
 
     // True if any stream carries a non-monotonic DTS
     public bool HasNonMonotonicDts => _nonMonotonicByStream.Count > 0;
 
+    // True when every stream carrying a non-monotonic DTS is audio, so the audio-only setts filter can
+    // repair it; a video or subtitle DTS is not audio-repairable (a video setts would reorder B-frames)
+    public bool NonMonotonicIsAudioOnly =>
+        HasNonMonotonicDts
+        && _nonMonotonicByStream.Keys.All(index =>
+            _codecTypeByStream.TryGetValue(index, out string? codecType)
+            && codecType.Equals("audio", StringComparison.OrdinalIgnoreCase)
+        );
+
     public void Add(FfMpegToolJsonSchema.Packet packet)
     {
-        // Fall back to PTS when DTS is absent, matching how the muxer derives DTS
-        double dts = !double.IsNaN(packet.DtsTime) ? packet.DtsTime : packet.PtsTime;
-        if (double.IsNaN(dts))
+        // Assess only packets that carry a real DTS. A missing DTS (e.g. Matroska video, which stores no
+        // DTS) is reconstructed by the muxer from PTS, whose display order is legitimately non-monotonic
+        // for B-frames, so it cannot be judged from packet data and must not be flagged
+        if (double.IsNaN(packet.DtsTime))
         {
             return;
         }
 
+        // Record the codec type so a flagged stream can be classified as audio or not
+        _codecTypeByStream[packet.StreamIndex] = packet.CodecType;
+
         // Flag a non-increasing DTS relative to the previous packet in the same stream
-        if (_lastDts.TryGetValue(packet.StreamIndex, out double previous) && dts <= previous)
+        if (
+            _lastDts.TryGetValue(packet.StreamIndex, out double previous)
+            && packet.DtsTime <= previous
+        )
         {
             _nonMonotonicByStream[packet.StreamIndex] =
                 _nonMonotonicByStream.GetValueOrDefault(packet.StreamIndex) + 1;
         }
-        _lastDts[packet.StreamIndex] = dts;
+        _lastDts[packet.StreamIndex] = packet.DtsTime;
     }
 }

PlexCleaner/FfMpegToolJsonSchema.cs

@@ -41,6 +41,29 @@ public class ClosedCaptionsTrack
         public int ClosedCaptions { get; set; }
     }
 
+    // Per-stream start and duration, used to verify a timestamp repair did not shift A/V sync
+    public class StreamTimingsProbe
+    {
+        [JsonPropertyName("streams")]
+        public List<StreamTiming> Streams { get; } = [];
+
+        public static StreamTimingsProbe FromJson(string json) =>
+            JsonSerializer.Deserialize(json, FfMpegToolJsonContext.Default.StreamTimingsProbe)
+            ?? throw new JsonException("Failed to deserialize StreamTimingsProbe");
+    }
+
+    public class StreamTiming
+    {
+        [JsonPropertyName("index")]
+        public int Index { get; set; }
+
+        [JsonPropertyName("start_time")]
+        public double StartTime { get; set; } = double.NaN;
+
+        [JsonPropertyName("duration")]
+        public double Duration { get; set; } = double.NaN;
+    }
+
     public class FormatInfo
     {
         [JsonPropertyName("format_name")]
@@ -172,4 +195,5 @@ public class Packet
 [JsonSerializable(typeof(FfMpegToolJsonSchema.FfProbe))]
 [JsonSerializable(typeof(FfMpegToolJsonSchema.Packet))]
 [JsonSerializable(typeof(FfMpegToolJsonSchema.ClosedCaptionsProbe))]
+[JsonSerializable(typeof(FfMpegToolJsonSchema.StreamTimingsProbe))]
 internal partial class FfMpegToolJsonContext : JsonSerializerContext;

PlexCleaner/FfProbeTool.cs

@@ -237,6 +237,53 @@ public bool GetClosedCaptions(string fileName, out bool hasClosedCaptions)
             return true;
         }
 
+        public bool GetStreamTimings(
+            string fileName,
+            out Dictionary<int, (double Start, double Duration)> timings
+        )
+        {
+            // Per-stream start and duration, used to verify a timestamp repair preserved A/V sync
+            timings = [];
+            Command command = GetBuilder()
+                .GlobalOptions(options => options.Default().HideBanner().LogLevelError())
+                .FfProbeOptions(options =>
+                    options
+                        .ShowStreams()
+                        .ShowEntries("stream=index,start_time,duration")
+                        .OutputFormatJson()
+                        .InputFile(fileName)
+                )
+                .Build();
+
+            // Execute command
+            Log.Debug("Getting stream timings : {FileName}", fileName);
+            if (!Execute(command, false, true, out BufferedCommandResult result))
+            {
+                return false;
+            }
+            if (result.ExitCode != 0)
+            {
+                Log.Error("Failed to get stream timings : {FileName}", fileName);
+                return LogFailedResult(result);
+            }
+
+            // FromJson throws on malformed output
+            try
+            {
+                FfMpegToolJsonSchema.StreamTimingsProbe probe =
+                    FfMpegToolJsonSchema.StreamTimingsProbe.FromJson(result.StandardOutput);
+                foreach (FfMpegToolJsonSchema.StreamTiming stream in probe.Streams)
+                {
+                    timings[stream.Index] = (stream.StartTime, stream.Duration);
+                }
+            }
+            catch (Exception e) when (Log.Logger.LogAndHandle(e))
+            {
+                return false;
+            }
+            return true;
+        }
+
         public bool GetAnalysisPackets(
             string fileName,
             Func<FfMpegToolJsonSchema.Packet, bool> packetFunc,

PlexCleaner/ProcessFile.cs

@@ -2214,12 +2214,13 @@ private bool RepairTimestampsAndSetState(ref bool modified)
 
     private bool TryLosslessTimestampRepair()
     {
-        // Only a demux-visible non-monotonic DTS can be rewritten losslessly, a post-decode-only break has
-        // no demux target and stays a reported failure, an analysis failure is treated as unrepaired too
+        // The audio-only setts filter can repair only an audio-stream DTS, so attempt it only when every
+        // non-monotonic stream is audio; a video or subtitle DTS, a post-decode-only break with no demux
+        // target, or an analysis failure all stay reported failures without a wasted rewrite
         if (
             !GetPacketAnalysis(false, out _, out DtsInfo? dtsInfo)
             || dtsInfo == null
-            || !dtsInfo.HasNonMonotonicDts
+            || !dtsInfo.NonMonotonicIsAudioOnly
         )
         {
             return false;
@@ -2253,13 +2254,52 @@ private bool TryLosslessTimestampRepair()
         return true;
     }
 
-    private static bool TimestampRepairRegressionGate(string original, string repaired) =>
-        // Lossless requires every stream's coded payload to be byte-identical before and after
-        // The streamhash muxer hashes packet data only, so a matching hash proves only timestamps changed
-        Tools.FfMpeg.GetStreamHashes(original, out Dictionary<int, string> before)
-        && Tools.FfMpeg.GetStreamHashes(repaired, out Dictionary<int, string> after)
+    // A timestamp nudge must not shift a stream's start or duration by more than the A/V-sync
+    // perceptibility threshold, so the repair never introduces audible drift
+    private const double SyncToleranceSeconds = 0.040;
+
+    private static bool TimestampRepairRegressionGate(string original, string repaired)
+    {
+        // Payload must be byte-identical; the streamhash muxer hashes packet data only, not timestamps,
+        // so a matching hash proves only the timestamps changed
+        if (
+            !Tools.FfMpeg.GetStreamHashes(original, out Dictionary<int, string> beforeHash)
+            || !Tools.FfMpeg.GetStreamHashes(repaired, out Dictionary<int, string> afterHash)
+            || beforeHash.Count != afterHash.Count
+            || !beforeHash.All(kvp =>
+                afterHash.TryGetValue(kvp.Key, out string? hash) && hash == kvp.Value
+            )
+        )
+        {
+            return false;
+        }
+
+        // Timing must be preserved; the streamhash proves the samples are identical but not where they
+        // play, so verify no stream's start or duration moved beyond the A/V-sync tolerance
+        return TimestampRepairSyncPreserved(original, repaired);
+    }
+
+    private static bool TimestampRepairSyncPreserved(string original, string repaired) =>
+        Tools.FfProbe.GetStreamTimings(
+            original,
+            out Dictionary<int, (double Start, double Duration)> before
+        )
+        && Tools.FfProbe.GetStreamTimings(
+            repaired,
+            out Dictionary<int, (double Start, double Duration)> after
+        )
         && before.Count == after.Count
-        && before.All(kvp => after.TryGetValue(kvp.Key, out string? hash) && hash == kvp.Value);
+        && before.All(kvp =>
+            after.TryGetValue(kvp.Key, out (double Start, double Duration) other)
+            && WithinSyncTolerance(kvp.Value.Start, other.Start)
+            && WithinSyncTolerance(kvp.Value.Duration, other.Duration)
+        );
+
+    private static bool WithinSyncTolerance(double before, double after) =>
+        // A missing value (NaN) cannot be compared, so treat it as unchanged rather than fail the repair
+        double.IsNaN(before)
+        || double.IsNaN(after)
+        || Math.Abs(after - before) <= SyncToleranceSeconds;
 
     public bool SetLastWriteTimeUtc(DateTime lastWriteTimeUtc)
     {

PlexCleanerTests/DtsInfoTests.cs

@@ -6,8 +6,17 @@ namespace PlexCleanerTests;
 
 public class DtsInfoTests
 {
-    private static FfMpegToolJsonSchema.Packet Packet(long streamIndex, double dtsTime) =>
-        new() { StreamIndex = streamIndex, DtsTime = dtsTime };
+    private static FfMpegToolJsonSchema.Packet Packet(
+        long streamIndex,
+        double dtsTime,
+        string codecType = "audio"
+    ) =>
+        new()
+        {
+            StreamIndex = streamIndex,
+            DtsTime = dtsTime,
+            CodecType = codecType,
+        };
 
     [Fact]
     public void Add_MonotonicDts_NoDetection()
@@ -58,4 +67,50 @@ public void Add_NanDts_Ignored()
 
         _ = dtsInfo.HasNonMonotonicDts.Should().BeFalse();
     }
+
+    [Fact]
+    public void NonMonotonicIsAudioOnly_AudioDts_True()
+    {
+        // A demux-visible audio DTS is repairable by the audio setts filter
+        DtsInfo dtsInfo = new();
+        dtsInfo.Add(Packet(1, 0.08, "audio"));
+        dtsInfo.Add(Packet(1, 0.08, "audio"));
+
+        _ = dtsInfo.NonMonotonicIsAudioOnly.Should().BeTrue();
+    }
+
+    [Fact]
+    public void NonMonotonicIsAudioOnly_VideoDts_False()
+    {
+        // A video DTS is not audio-repairable, a video setts would reorder B-frames
+        DtsInfo dtsInfo = new();
+        dtsInfo.Add(Packet(0, 0.08, "video"));
+        dtsInfo.Add(Packet(0, 0.08, "video"));
+
+        _ = dtsInfo.HasNonMonotonicDts.Should().BeTrue();
+        _ = dtsInfo.NonMonotonicIsAudioOnly.Should().BeFalse();
+    }
+
+    [Fact]
+    public void NonMonotonicIsAudioOnly_MixedAudioAndVideo_False()
+    {
+        // If any offending stream is non-audio the audio setts cannot fully repair the file
+        DtsInfo dtsInfo = new();
+        dtsInfo.Add(Packet(0, 0.08, "video"));
+        dtsInfo.Add(Packet(0, 0.08, "video"));
+        dtsInfo.Add(Packet(1, 0.08, "audio"));
+        dtsInfo.Add(Packet(1, 0.08, "audio"));
+
+        _ = dtsInfo.NonMonotonicIsAudioOnly.Should().BeFalse();
+    }
+
+    [Fact]
+    public void NonMonotonicIsAudioOnly_NoDts_False()
+    {
+        DtsInfo dtsInfo = new();
+        dtsInfo.Add(Packet(1, 0.04, "audio"));
+        dtsInfo.Add(Packet(1, 0.08, "audio"));
+
+        _ = dtsInfo.NonMonotonicIsAudioOnly.Should().BeFalse();
+    }
 }

PlexCleanerTests/StreamTimingsProbeTests.cs

@@ -0,0 +1,40 @@
+using AwesomeAssertions;
+using PlexCleaner;
+using Xunit;
+
+namespace PlexCleanerTests;
+
+public class StreamTimingsProbeTests
+{
+    [Fact]
+    public void FromJson_StartAndDuration_Parsed()
+    {
+        // ffprobe emits start_time and duration as strings, parsed to double
+        // lang=json
+        const string json = """
+            { "streams": [ { "index": 0, "start_time": "0.000000", "duration": "1234.567000" } ] }
+            """;
+        FfMpegToolJsonSchema.StreamTimingsProbe probe =
+            FfMpegToolJsonSchema.StreamTimingsProbe.FromJson(json);
+
+        _ = probe.Streams.Should().ContainSingle();
+        _ = probe.Streams[0].Index.Should().Be(0);
+        _ = probe.Streams[0].StartTime.Should().Be(0.0);
+        _ = probe.Streams[0].Duration.Should().Be(1234.567);
+    }
+
+    [Fact]
+    public void FromJson_MissingTiming_DefaultsToNaN()
+    {
+        // A stream without start_time or duration keeps the NaN sentinel, treated as unchanged by the gate
+        // lang=json
+        const string json = """
+            { "streams": [ { "index": 1 } ] }
+            """;
+        FfMpegToolJsonSchema.StreamTimingsProbe probe =
+            FfMpegToolJsonSchema.StreamTimingsProbe.FromJson(json);
+
+        _ = double.IsNaN(probe.Streams[0].StartTime).Should().BeTrue();
+        _ = double.IsNaN(probe.Streams[0].Duration).Should().BeTrue();
+    }
+}