From 944e11d88576e16e4d68f539fe1a8c82132dc47e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Jul 2026 05:45:48 +0000 Subject: [PATCH 01/19] Bump the nuget-deps group with 1 update (#831) Bumps ptr727.LanguageTags from 1.5.33 to 1.5.39 --- updated-dependencies: - dependency-name: ptr727.LanguageTags dependency-version: 1.5.39 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: nuget-deps ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Directory.Packages.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 5c431531..63774c59 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -6,7 +6,7 @@ - + From ecf952ac2018a1a3d47477d4e1069a260db7dcb1 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Mon, 13 Jul 2026 11:36:43 -0700 Subject: [PATCH 02/19] Add lossless DTS timestamp repair and verify reclassification (#833) Verify classifies -f null stderr (fail-closed, streamed): a non-monotonic DTS is a correctable failure repaired losslessly with the setts bitstream filter (byte-identical gate) ahead of the re-encode tier for genuine decode corruption. Consolidates the bitrate/DTS packet pass, switches closed-caption detection to ffprobe analyze_frames, and adds the DtsTimestampRepair example plugin. See #827. --- HISTORY.md | 13 + PlexCleaner.slnx | 1 + PlexCleaner/DtsInfo.cs | 34 +++ PlexCleaner/FfMpegBuilder.cs | 5 + PlexCleaner/FfMpegTool.cs | 123 ++++++++- PlexCleaner/FfMpegToolJsonSchema.cs | 17 ++ PlexCleaner/FfProbeBuilder.cs | 16 +- PlexCleaner/FfProbeTool.cs | 120 ++++----- PlexCleaner/MediaTool.cs | 55 ++++ PlexCleaner/ProcessFile.cs | 237 ++++++++++++++---- PlexCleaner/Program.cs | 5 +- PlexCleaner/VerifyClassifier.cs | 60 +++++ PlexCleaner/VerifyResult.cs | 13 + PlexCleanerTests/ClosedCaptionsProbeTests.cs | 49 ++++ PlexCleanerTests/DtsInfoTests.cs | 61 +++++ PlexCleanerTests/FileNameEscapingTests.cs | 25 -- PlexCleanerTests/PlexCleanerTests.csproj | 3 +- PlexCleanerTests/PluginLoaderTests.cs | 13 + PlexCleanerTests/VerifyClassifierTests.cs | 69 +++++ .../DtsTimestampRepair.csproj | 10 + .../DtsTimestampRepairPlugin.cs | 90 +++++++ README.md | 14 +- cspell.json | 5 + version.json | 2 +- 24 files changed, 864 insertions(+), 176 deletions(-) create mode 100644 PlexCleaner/DtsInfo.cs create mode 100644 PlexCleaner/VerifyClassifier.cs create mode 100644 PlexCleaner/VerifyResult.cs create mode 100644 PlexCleanerTests/ClosedCaptionsProbeTests.cs create mode 100644 PlexCleanerTests/DtsInfoTests.cs delete mode 100644 PlexCleanerTests/FileNameEscapingTests.cs create mode 100644 PlexCleanerTests/VerifyClassifierTests.cs create mode 100644 Plugins/DtsTimestampRepair/DtsTimestampRepair.csproj create mode 100644 Plugins/DtsTimestampRepair/DtsTimestampRepairPlugin.cs diff --git a/HISTORY.md b/HISTORY.md index 6f50665d..5a9feb97 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -4,6 +4,19 @@ Utility to optimize media files for Direct Play in Plex, Emby, Jellyfin, etc. ## Release History +- Version 3.21: + - Repair non-monotonic DTS muxer warnings losslessly instead of failing repair permanently. + - `ffmpeg -f null` can exit `0` yet emit `Application provided invalid, non monotonically increasing dts to muxer` for files that may decode and play correctly. + - The previous "any stderr means failure" rule promoted this muxer-interleaving artifact to a hard `VerifyFailed`/`RepairFailed`, and a re-encode could not fix it because Matroska stores no DTS and ffmpeg re-derives a non-monotonic timeline on read. + - Verify now classifies the decode diagnostics deterministically as clean, a benign timestamp-only failure, or a decode error; the timestamp-only failure is correctable rather than permanent, and everything else fails (fail-closed, so an unrecognized diagnostic fails as a decode error). + - The classification streams the output line by line, so memory stays bounded even when a file emits a warning per packet ([#827](https://github.com/ptr727/PlexCleaner/issues/827)). + - Added a lossless timestamp repair as the first repair tier. + - When verification detects a demux-visible non-monotonic DTS, the audio packet timestamps are rewritten to be strictly monotonic using the `setts` bitstream filter with a stream copy (no re-encode), then re-verified. + - A regression gate compares the per-stream coded payload hash before and after and discards the result unless every stream is byte-identical, so the repair can never alter the media. The full re-encode repair remains for genuine decode corruption. + - Consolidated the bitrate and DTS packet analyses into a single `ffprobe -show_packets` pass, computing the per-second bitrate and the per-stream DTS monotonicity together instead of reading packets twice. + - Switched closed caption detection to `ffprobe -analyze_frames -show_entries stream=closed_captions`, replacing the `movie=...[out0+subcc]` lavfi filter and its QuickScan snippet-remux workaround; QuickScan now bounds the scan with `-read_intervals`. + - Added the `DtsTimestampRepair` example plugin. + - It revisits files that a previous version marked `RepairFailed`, re-verifies them, clears the flag when the only problem was timestamps, and losslessly repairs the timestamps when the DTS is demux-visible. Not available in AOT builds. - Version 3.20: - Switched tool downloads and the application version check to the resilient HTTP client in `ptr727.Utilities` (retry with backoff and a circuit breaker via `Microsoft.Extensions.Http.Resilience`), replacing the plain `HttpClient`. - Enabled closed caption removal for H.265/HEVC video: the SEI NAL unit lookup keyed on `h265` never matched FFprobe's `hevc` codec name, so HEVC files were incorrectly reported as an "Unsupported video format for Closed Captions removal". HEVC video (excluding HDR10 and HDR10+ content, which remains guarded) is now cleaned using the `filter_units=remove_types=39` bitstream filter, same as H.264 and MPEG-2. diff --git a/PlexCleaner.slnx b/PlexCleaner.slnx index 223749ed..05df2d7c 100644 --- a/PlexCleaner.slnx +++ b/PlexCleaner.slnx @@ -77,6 +77,7 @@ + diff --git a/PlexCleaner/DtsInfo.cs b/PlexCleaner/DtsInfo.cs new file mode 100644 index 00000000..705a0e5e --- /dev/null +++ b/PlexCleaner/DtsInfo.cs @@ -0,0 +1,34 @@ +namespace PlexCleaner; + +public sealed class DtsInfo +{ + // Last DTS seen per stream index + private readonly Dictionary _lastDts = []; + + // Count of non-monotonic packets per stream index + private readonly Dictionary _nonMonotonicByStream = []; + + // Stream indexes carrying a non-monotonic DTS, with the per-stream count + public IReadOnlyDictionary NonMonotonicByStream => _nonMonotonicByStream; + + // True if any stream carries a non-monotonic DTS + public bool HasNonMonotonicDts => _nonMonotonicByStream.Count > 0; + + public void Add(FfMpegToolJsonSchema.Packet packet) + { + // Fall back to PTS when DTS is absent, matching how the muxer derives DTS + double dts = !double.IsNaN(packet.DtsTime) ? packet.DtsTime : packet.PtsTime; + if (double.IsNaN(dts)) + { + return; + } + + // Flag a non-increasing DTS relative to the previous packet in the same stream + if (_lastDts.TryGetValue(packet.StreamIndex, out double previous) && dts <= previous) + { + _nonMonotonicByStream[packet.StreamIndex] = + _nonMonotonicByStream.GetValueOrDefault(packet.StreamIndex) + 1; + } + _lastDts[packet.StreamIndex] = dts; + } +} diff --git a/PlexCleaner/FfMpegBuilder.cs b/PlexCleaner/FfMpegBuilder.cs index 1a42c134..b60c3f8b 100644 --- a/PlexCleaner/FfMpegBuilder.cs +++ b/PlexCleaner/FfMpegBuilder.cs @@ -182,6 +182,11 @@ public class OutputOptions(ArgumentsBuilder argumentsBuilder) public OutputOptions BitstreamFilterVideo(string option) => BitstreamFilterVideo().Add(option); + public OutputOptions BitstreamFilterAudio() => Add("-bsf:a"); + + public OutputOptions BitstreamFilterAudio(string option) => + BitstreamFilterAudio().Add(option); + public OutputOptions SeekStartStop(TimeSpan timeStart, TimeSpan timeStop) => timeStart == TimeSpan.Zero || timeStop == TimeSpan.Zero ? this diff --git a/PlexCleaner/FfMpegTool.cs b/PlexCleaner/FfMpegTool.cs index f14444e9..f0776e93 100644 --- a/PlexCleaner/FfMpegTool.cs +++ b/PlexCleaner/FfMpegTool.cs @@ -152,7 +152,7 @@ public override bool Update(string updateFile) return true; } - public bool VerifyMedia(string fileName) + public VerifyResult VerifyMedia(string fileName) { // Build command line Command command = GetBuilder() @@ -162,12 +162,44 @@ public bool VerifyMedia(string fileName) .OutputOptions(options => options.Default().NullOutput()) .Build(); - // Execute command; ffmpeg can exit 0 yet still report stream errors on stderr, treat any stderr as failure - return Execute(command, true, true, out BufferedCommandResult result) - && ( - (result.ExitCode == 0 && result.StandardError.Trim().Length == 0) - || LogFailedResult(result) - ); + // Execute command: ffmpeg can exit 0 yet report stream errors on stderr + // Classify stderr line by line as it streams to keep memory bounded, e.g. non-monotonic-DTS file emits a warning per packet + VerifyClassifier.Accumulator classifier = new(); + if (!ExecuteStreamStdErr(command, classifier.Add, out int exitCode)) + { + // Process could not run + return VerifyResult.DecodeError; + } + + // A non-zero exit is always a failure, fail closed even if stderr shows only the timestamp warning + VerifyResult verifyResult = classifier.Result; + if (exitCode != 0) + { + verifyResult = VerifyResult.DecodeError; + } + if (verifyResult == VerifyResult.DecodeError) + { + // A silent non-zero exit has no error line, omit the empty field rather than logging blank + string error = CleanForLog(classifier.FirstError ?? string.Empty); + if (string.IsNullOrEmpty(error)) + { + Log.Error( + "Failed execution of {ToolType} : ExitCode: {ExitCode}", + GetToolType(), + exitCode + ); + } + else + { + Log.Error( + "Failed execution of {ToolType} : ExitCode: {ExitCode} : {Error}", + GetToolType(), + exitCode, + error + ); + } + } + return verifyResult; } public bool ReMuxToMkv(string inputName, string outputName) => @@ -320,6 +352,83 @@ public bool ConvertToMkv(string inputName, string outputName) && (result.ExitCode == 0 || LogFailedResult(result)); } + public bool SetTimestamps(string inputName, string outputName) + { + // Losslessly rewrite audio packet timestamps to be strictly monotonic using the setts bitstream filter + // https://ffmpeg.org/ffmpeg-bitstream-filters.html#setts + // Audio only, the expression forces PTS monotonic which is safe where PTS equals DTS, applying + // it to video would reorder B-frames, a video-only DTS break instead fails the caller's re-verify + + // Delete output file + File.Delete(outputName); + + // Build command line, the escaped comma separates the setts option arguments + Command command = GetBuilder() + .GlobalOptions(options => options.Default()) + .InputOptions(options => options.Default().TestSnippets().InputFile(inputName)) + .OutputOptions(options => + options + .MapAllCodecCopy() + .BitstreamFilterAudio( + "\"setts=pts=max(PTS\\,PREV_OUTPTS+1):dts=max(DTS\\,PREV_OUTDTS+1)\"" + ) + .Default() + .FormatMatroska() + .OutputFile(outputName) + ) + .Build(); + + // Execute command + return Execute(command, true, true, out BufferedCommandResult result) + && (result.ExitCode == 0 || LogFailedResult(result)); + } + + public bool GetStreamHashes(string fileName, out Dictionary streamHashes) + { + // Streamhash muxer hashes payload only, not timestamps, so an identical hash proves setts changed timestamps and nothing else + streamHashes = []; + + // Build command line, stream copy to the streamhash muxer written to stdout + Command command = GetBuilder() + .GlobalOptions(options => options.Default()) + .InputOptions(options => options.Default().InputFile(fileName)) + .OutputOptions(options => + options.MapAllCodecCopy().Format("streamhash").Add("-hash").Add("md5").Add("-") + ) + .Build(); + + // Execute command, capturing full stdout + if (!Execute(command, false, true, out BufferedCommandResult result)) + { + return false; + } + if (result.ExitCode != 0) + { + return LogFailedResult(result); + } + + // Parse lines of the form "index,type,md5=value" + foreach ( + string line in result.StandardOutput.Split( + '\n', + StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries + ) + ) + { + string[] parts = line.Split(',', 3); + if ( + parts.Length == 3 + && int.TryParse(parts[0], out int index) + && !streamHashes.ContainsKey(index) + ) + { + // Keep type and hash so a stream reorder or payload change is detected + streamHashes[index] = $"{parts[1]},{parts[2]}"; + } + } + return streamHashes.Count > 0; + } + public bool RemoveNalUnits(string inputName, int nalUnit, string outputName) { // Remove SEI NAL units e.g. EIA-608 and CTA-708 content diff --git a/PlexCleaner/FfMpegToolJsonSchema.cs b/PlexCleaner/FfMpegToolJsonSchema.cs index 1aeb8903..473685fb 100644 --- a/PlexCleaner/FfMpegToolJsonSchema.cs +++ b/PlexCleaner/FfMpegToolJsonSchema.cs @@ -25,6 +25,22 @@ public static FfProbe FromJson(string json) => ?? throw new JsonException("Failed to deserialize FfProbe"); } + public class ClosedCaptionsProbe + { + [JsonPropertyName("streams")] + public List Streams { get; } = []; + + public static ClosedCaptionsProbe FromJson(string json) => + JsonSerializer.Deserialize(json, FfMpegToolJsonContext.Default.ClosedCaptionsProbe) + ?? throw new JsonException("Failed to deserialize ClosedCaptionsProbe"); + } + + public class ClosedCaptionsTrack + { + [JsonPropertyName("closed_captions")] + public int ClosedCaptions { get; set; } + } + public class FormatInfo { [JsonPropertyName("format_name")] @@ -155,4 +171,5 @@ public class Packet )] [JsonSerializable(typeof(FfMpegToolJsonSchema.FfProbe))] [JsonSerializable(typeof(FfMpegToolJsonSchema.Packet))] +[JsonSerializable(typeof(FfMpegToolJsonSchema.ClosedCaptionsProbe))] internal partial class FfMpegToolJsonContext : JsonSerializerContext; diff --git a/PlexCleaner/FfProbeBuilder.cs b/PlexCleaner/FfProbeBuilder.cs index b33ee588..64d67778 100644 --- a/PlexCleaner/FfProbeBuilder.cs +++ b/PlexCleaner/FfProbeBuilder.cs @@ -8,19 +8,6 @@ namespace PlexCleaner; public partial class FfProbe { - public static string EscapeMovieFileName(string fileName) => - // Escape the file name so that it does not interfere with building the filter graph - // https://superuser.com/questions/1893137/how-to-quote-a-file-name-containing-single-quotes-in-ffmpeg-ffprobe-movie-filena - // See av_get_token() in https://github.com/FFmpeg/FFmpeg/blob/master/libavutil/avstring.c - fileName - .Replace(@"\", @"/") - .Replace(@":", @"\\:") - .Replace(@"'", @"\\\'") - .Replace(@",", @"\\\,") - .Replace(@";", @"\\\;") - .Replace(@"[", @"\\\[") - .Replace(@"]", @"\\\]"); - public class GlobalOptions(ArgumentsBuilder argumentsBuilder) { public GlobalOptions Default() => AnalyzeDuration("2G").ProbeSize("2G"); @@ -110,6 +97,9 @@ public FfProbeOptions SeekStop(TimeSpan timeSpan) => public FfProbeOptions QuickScan() => Program.Options.QuickScan ? SeekStop(Program.QuickScanTimeSpan) : this; + public FfProbeOptions ReadIntervalFrames(int frames) => + frames <= 0 ? this : Add("-read_intervals").Add($"%+#{frames}"); + public FfProbeOptions InputFile(string option) => Add($"\"{option}\""); public FfProbeOptions Add(string option) => Add(option, false); diff --git a/PlexCleaner/FfProbeTool.cs b/PlexCleaner/FfProbeTool.cs index a3a745c1..e89b1414 100644 --- a/PlexCleaner/FfProbeTool.cs +++ b/PlexCleaner/FfProbeTool.cs @@ -1,4 +1,3 @@ -using System.Diagnostics; using System.Text; using System.Text.Json; using System.Text.Json.Serialization; @@ -192,107 +191,75 @@ out string error } } - public bool GetSubCcPackets( - string fileName, - Func packetFunc - ) + public bool GetClosedCaptions(string fileName, out bool hasClosedCaptions) { - // TODO: Switch to ffprobe and analyze_frames (when available in the shipping version). - // `ffprobe -i FILE -show_entries stream=closed_captions -select_streams v:0 -analyze_frames -read_intervals %X` - - // Quickscan is not supported with subcc filter - // -t and read_intervals do not work with the subcc filter - // https://superuser.com/questions/1893673/how-to-time-limit-the-input-stream-duration-when-using-movie-filenameout0subcc - // ReMux using FFmpeg to a snippet file then scan the snippet file - Command command; - if (Program.Options.QuickScan) - { - // Keep in sync with FfMpegTool.ReMuxToFormat() - - // Create a temp filename based on the input name - string tempName = Path.ChangeExtension(fileName, ".tmp13"); - Debug.Assert(fileName != tempName); - File.Delete(tempName); - - // Use Matroska for snippet format as it supports more stream formats - // E.g. DVCPRO video streams can be muxed into MKV but not into TS - // [mpegts @ 000001543cf744c0] Stream 0, codec dvvideo, is muxed as a private data stream and may not be recognized upon reading. - - // Build command line - command = Tools - .FfMpeg.GetBuilder() - .GlobalOptions(options => options.Default()) - .InputOptions(options => - options.Default().SeekStop(Program.QuickScanTimeSpan).InputFile(fileName) - ) - .OutputOptions(options => - options.MapAllCodecCopy().Default().FormatMatroska().OutputFile(tempName) - ) - .Build(); - - // Execute command - Log.Debug("Creating temp media file : {TempFileName}", tempName); - if (!Tools.FfMpeg.Execute(command, true, true, out BufferedCommandResult result)) - { - Log.Error("Failed to create temp media file : {TempFileName}", tempName); - LogErrorOutput(result.StandardError.Trim()); - File.Delete(tempName); - return false; - } - - // Use the temp file as the input file - fileName = tempName; - } - - // Build command line - // Get packet info using subcc filter - // https://www.ffmpeg.org/ffmpeg-devices.html#Options-10 - command = GetBuilder() + // Detect EIA-608/CTA-708 closed captions embedded in the video stream + // analyze_frames decodes frames to surface the stream-level closed_captions flag + hasClosedCaptions = false; + Command command = GetBuilder() .GlobalOptions(options => options.Default().HideBanner().LogLevelError()) .FfProbeOptions(options => options - .SelectStreams("s:0") - .Format("lavfi") - .Input($"\"movie={EscapeMovieFileName(fileName)}[out0+subcc]\"") - .ShowPackets() + .SelectStreams("v:0") + .AnalyzeFrames() + .ReadIntervalFrames( + Program.Options.QuickScan ? Program.QuickScanFrameCount : 0 + ) + .ShowEntries("stream=closed_captions") .OutputFormatJson() + .InputFile(fileName) ) .Build(); - // Get packet list - Log.Debug("Getting subcc packet info : {FileName}", fileName); - bool ret = GetPackets(command, packetFunc, out string error); - if (!ret) + // Execute command + Log.Debug("Getting closed caption info : {FileName}", fileName); + if (!Execute(command, false, true, out BufferedCommandResult result)) { - Log.Error("Failed to get subcc packet info : {FileName}", fileName); - LogErrorOutput(error); + return false; } - if (Program.Options.QuickScan) + if (result.ExitCode != 0) { - // Delete the temp file - File.Delete(fileName); + Log.Error("Failed to get closed caption info : {FileName}", fileName); + return LogFailedResult(result); } - return ret; + + // Any video stream reporting closed captions, FromJson throws on malformed output + try + { + FfMpegToolJsonSchema.ClosedCaptionsProbe probe = + FfMpegToolJsonSchema.ClosedCaptionsProbe.FromJson(result.StandardOutput); + hasClosedCaptions = probe.Streams.Any(stream => stream.ClosedCaptions != 0); + } + catch (Exception e) when (Log.Logger.LogAndHandle(e)) + { + return false; + } + return true; } - public bool GetBitratePackets( + public bool GetAnalysisPackets( string fileName, - Func packetFunc + Func packetFunc, + bool quickScan ) { // Build command line Command command = GetBuilder() .GlobalOptions(options => options.Default().HideBanner().LogLevelError()) .FfProbeOptions(options => - options.QuickScan().ShowPackets().OutputFormatJson().InputFile(fileName) + options + .SeekStop(quickScan ? Program.QuickScanTimeSpan : TimeSpan.Zero) + .ShowPackets() + .OutputFormatJson() + .InputFile(fileName) ) .Build(); // Get packet list - Log.Debug("Getting bitrate packets : {FileName}", fileName); + Log.Debug("Getting analysis packets : {FileName}", fileName); if (!GetPackets(command, packetFunc, out string error)) { - Log.Error("Failed to get bitrate packets : {FileName}", fileName); + Log.Error("Failed to get analysis packets : {FileName}", fileName); LogErrorOutput(error); return false; } @@ -308,8 +275,7 @@ public bool GetMediaProps(string fileName, out MediaProps mediaProps) public bool GetMediaPropsJson(string fileName, out string json) { - // TODO: Add analyze_frames when available in all FFmpeg builds - // https://github.com/FFmpeg/FFmpeg/commit/90af8e07b02e690a9fe60aab02a8bccd2cbf3f01 + // Do not use analyze_frames, it would add closed_captions, film_grain, nb_read_frames but forces full decode of every stream // Build command line json = string.Empty; diff --git a/PlexCleaner/MediaTool.cs b/PlexCleaner/MediaTool.cs index 91862241..d75df70e 100644 --- a/PlexCleaner/MediaTool.cs +++ b/PlexCleaner/MediaTool.cs @@ -248,6 +248,61 @@ out BufferedCommandResult bufferedCommandResult } } + public bool ExecuteStreamStdErr(Command command, Action lineAction, out int exitCode) + { + exitCode = -1; + int processId = -1; + try + { + // Stream stderr line by line to the caller instead of buffering it + PipeTarget stdErrTarget = PipeTarget.Create( + async (stream, cancellationToken) => + { + using StreamReader reader = new(stream, Encoding.Default, false, 1024, true); + while (await reader.ReadLineAsync(cancellationToken) is { } line) + { + if (cancellationToken.IsCancellationRequested) + { + return; + } + lineAction(line); + } + } + ); + + CommandTask task = command + .WithStandardOutputPipe(PipeTarget.Null) + .WithStandardErrorPipe(stdErrTarget) + .WithValidation(CommandResultValidation.None) + .ExecuteAsync(CancellationToken.None, Program.CancelToken()); + processId = task.ProcessId; + Log.Debug( + "Executing {ToolType} : ProcessId: {ProcessId}, Arguments: {Arguments}", + GetToolType(), + processId, + command.Arguments + ); + + CommandResult commandResult = task.Task.GetAwaiter().GetResult(); + exitCode = commandResult.ExitCode; + return task.Task.IsCompletedSuccessfully; + } + catch (OperationCanceledException) + { + Log.Error( + "Cancelled execution of {ToolType} : ProcessId: {ProcessId}, Arguments: {Arguments}", + GetToolType(), + processId, + command.Arguments + ); + return false; + } + catch (Exception e) when (Log.Logger.LogAndHandle(e)) + { + return false; + } + } + public static PipeTarget ToStringBuilder(StringBuilder stringBuilder) => PipeTarget.Create( async (stream, cancellationToken) => diff --git a/PlexCleaner/ProcessFile.cs b/PlexCleaner/ProcessFile.cs index 3acbde2c..0f286739 100644 --- a/PlexCleaner/ProcessFile.cs +++ b/PlexCleaner/ProcessFile.cs @@ -23,6 +23,9 @@ public class ProcessFile private SidecarFile _sidecarFile; + // Classification of the most recent stream verify, used to choose the repair strategy + private VerifyResult _lastVerifyResult; + public ProcessFile(string mediaFile) { FileInfo = new FileInfo(mediaFile); @@ -1224,7 +1227,7 @@ private bool FindClosedCaptionTracks(bool conditional, out VideoProps? videoProp return true; } - // Running lavfi is expensive, skip if already verified or closed captions already removed + // Running analyze_frames is expensive, skip if already verified or closed captions already removed if ( conditional && ( @@ -1238,20 +1241,9 @@ private bool FindClosedCaptionTracks(bool conditional, out VideoProps? videoProp return true; } - // Get packet info using ccsub filter - bool packetsFound = false; + // Detect closed captions embedded in the video stream Log.Information("Finding Closed Captions in video stream : {FileName}", FileInfo.FullName); - if ( - !Tools.FfProbe.GetSubCcPackets( - FileInfo.FullName, - _ => - { - // Stop processing more packets - packetsFound = true; - return false; - } - ) - ) + if (!Tools.FfProbe.GetClosedCaptions(FileInfo.FullName, out bool hasClosedCaptions)) { // Error Log.Error( @@ -1261,8 +1253,8 @@ private bool FindClosedCaptionTracks(bool conditional, out VideoProps? videoProp return false; } - // Any packets means there are subtitles present in the video stream - if (packetsFound) + // Mark the first video track when captions are present + if (hasClosedCaptions) { // Use the first video track from FfProbe videoProps = FfProbeProps.Video.First(); @@ -1757,32 +1749,37 @@ public bool Verify(bool conditional, out bool canRepair) // Will update sidecar state if bitrate exceeded _ = VerifyBitrate(); - // Verify media streams, repair is possible + // Verify media streams, both failure kinds are repairable canRepair = true; - return VerifyMediaStreams(FileInfo); + _lastVerifyResult = VerifyMediaStreams(FileInfo); + + // Deterministic: pass only when clean, a timestamp-only or decode result is a repairable failure + return _lastVerifyResult == VerifyResult.Clean; } - public static bool VerifyMediaStreams(FileInfo fileInfo) + public static VerifyResult VerifyMediaStreams(FileInfo fileInfo) { // Verify Log.Debug("Verifying media streams : {FileName}", fileInfo.FullName); - if (!Tools.FfMpeg.VerifyMedia(fileInfo.FullName)) + VerifyResult verifyResult = Tools.FfMpeg.VerifyMedia(fileInfo.FullName); + + // Log the classified outcome so a failure is diagnosable, unless it was a cancellation + if (!Program.IsCancelledError()) { - // Cancel requested - if (Program.IsCancelledError()) + if (verifyResult == VerifyResult.DecodeError) { - return false; + Log.Error("Failed to verify media streams : {FileName}", fileInfo.FullName); + } + else if (verifyResult == VerifyResult.TimestampOnly) + { + // Correctable failure, the decision Warning is emitted later if the repair runs + Log.Information( + "Verify detected non-monotonic DTS timestamps : {FileName}", + fileInfo.FullName + ); } - - // Failed stream validation - Log.Error("Failed to verify media streams : {FileName}", fileInfo.FullName); - - // Caller should update the state - return false; } - - // Verified - return true; + return verifyResult; } public bool DeleteFailedFile() @@ -1861,6 +1858,13 @@ public bool VerifyAndRepair(ref bool modified) Debug.Assert(!_sidecarFile.State.HasFlag(SidecarFile.StatesType.Verified)); Debug.Assert(!_sidecarFile.State.HasFlag(SidecarFile.StatesType.Repaired)); + // Non-monotonic DTS is a repairable failure, fix it losslessly with setts + // A re-encode cannot fix timestamps so it is not a fallback here + if (_lastVerifyResult == VerifyResult.TimestampOnly) + { + return RepairTimestampsAndSetState(ref modified); + } + // Attempt repair, if repair fails the original file will not be modified bool repaired = RepairAndReVerify(); @@ -2056,22 +2060,12 @@ private bool RepairAndReVerify() // [h264 @ 000002a21166bd00] Invalid NAL unit size (-1148261185 > 8772). // [matroska,webm @ 0000029a256d9280] Length 7 indicated by an EBML number's first byte 0x02 at pos 1601277 (0x186efd) exceeds max length 4. - // TODO: Can we ignore the monotonically increasing display time stamp issue? - // Lots of similar reports, can't find a CLI option to disable or ignore this as an error - // Also see FFmpeg AVFMT_TS_NONSTRICT option - // [null @ 0000018cd6bf1800] Application provided invalid, non monotonically increasing dts to muxer in stream 0: 8 >= 8 - // [null @ 0000018cd6bf1800] Application provided invalid, non monotonically increasing dts to muxer in stream 0: 12 >= 12 - // [null @ 0000018cd6bf1800] Application provided invalid, non monotonically increasing dts to muxer in stream 0: 16 >= 16 - // [null @ 0000018cd6bf1800] Application provided invalid, non monotonically increasing dts to muxer in stream 0: 20 >= 20 - // [null @ 0000018cd6bf1800] Application provided invalid, non monotonically increasing dts to muxer in stream 0: 348 >= 348 - // TODO: HandBrake sometimes fails with what looks like a remux error // ERROR: avformatMux: track 1, av_interleaved_write_frame failed with error 'Invalid argument' // M[15:35:43] libhb: work result = 4 // TODO: FfMpeg fails to decode some files // https://trac.ffmpeg.org/search?q=%22Invalid+NAL+unit+size%22&noquickjump=1&milestone=on&ticket=on&wiki=on - // https://trac.ffmpeg.org/search?q=%22non+monotonically+increasing+dts+to+muxer%22&noquickjump=1&milestone=on&ticket=on&wiki=on // TODO: FfMpeg x265 requires input resolution to be multiple of chroma subsampling // https://stackoverflow.com/questions/50371919/ffmpeg-cannot-open-libx265-encoder-error-initializing-output-stream-00-err @@ -2135,8 +2129,9 @@ private bool RepairAndReVerify() return false; } - // Re-encoding succeeded, re-verify the temp file - if (!VerifyMediaStreams(new FileInfo(tempName))) + // Require a clean re-verify, accepting a timestamp-only result would mark a file Verified that + // still fails verification, and a future run would skip it as already verified + if (VerifyMediaStreams(new FileInfo(tempName)) != VerifyResult.Clean) { // Failed File.Delete(tempName); @@ -2153,6 +2148,139 @@ private bool RepairAndReVerify() return true; } + public bool RepairTimestamps(ref bool modified) + { + // Only process Matroska files, the audio timestamp repair does not require a video stream + if (!SidecarFile.IsMkvFile(FileInfo.FullName)) + { + return true; + } + + // Classify the current verify state + _lastVerifyResult = VerifyMediaStreams(FileInfo); + + // Cancel requested + if (Program.IsCancelled()) + { + return false; + } + + switch (_lastVerifyResult) + { + case VerifyResult.Clean: + // Nothing to repair, clear any stale failure flags and mark verified + _sidecarFile.State |= SidecarFile.StatesType.Verified; + _sidecarFile.State &= ~SidecarFile.StatesType.VerifyFailed; + _sidecarFile.State &= ~SidecarFile.StatesType.RepairFailed; + return Refresh(false); + case VerifyResult.TimestampOnly: + // Benign non-monotonic DTS, clean losslessly when demux-visible + return RepairTimestampsAndSetState(ref modified); + case VerifyResult.DecodeError: + // Genuine decode corruption, not repairable here, leave the state unchanged + return true; + default: + throw new NotImplementedException(); + } + } + + private enum TimestampRepair + { + // No demux-visible break, the muxer warning is a post-decode artifact with no defect to repair + NotApplicable, + + // Timestamps rewritten losslessly + Repaired, + + // A demux-visible break, but the lossless repair could not complete + Failed, + } + + private bool RepairTimestampsAndSetState(ref bool modified) + { + switch (TryLosslessTimestampRepair()) + { + case TimestampRepair.Repaired: + _sidecarFile.State |= SidecarFile.StatesType.Verified; + _sidecarFile.State &= ~SidecarFile.StatesType.VerifyFailed; + _sidecarFile.State |= SidecarFile.StatesType.Repaired; + _sidecarFile.State &= ~SidecarFile.StatesType.RepairFailed; + modified = true; + return Refresh(true); + case TimestampRepair.NotApplicable: + // No demux-visible defect, the media is benign, mark verified without a rewrite + _sidecarFile.State |= SidecarFile.StatesType.Verified; + _sidecarFile.State &= ~SidecarFile.StatesType.VerifyFailed; + _sidecarFile.State &= ~SidecarFile.StatesType.RepairFailed; + return Refresh(false); + case TimestampRepair.Failed: + // Do not touch state on cancellation, the caller retries next run + if (Program.IsCancelled()) + { + return false; + } + // The lossless repair failed, leave it as a repair failure + _sidecarFile.State |= SidecarFile.StatesType.VerifyFailed; + _sidecarFile.State &= ~SidecarFile.StatesType.Verified; + _sidecarFile.State |= SidecarFile.StatesType.RepairFailed; + _sidecarFile.State &= ~SidecarFile.StatesType.Repaired; + _ = Refresh(false); + return false; + default: + throw new NotImplementedException(); + } + } + + private TimestampRepair TryLosslessTimestampRepair() + { + // Could not analyze packets, treat as a repair failure rather than assume benign + if (!GetPacketAnalysis(false, out _, out DtsInfo? dtsInfo) || dtsInfo == null) + { + return TimestampRepair.Failed; + } + + // No demux-visible break, a post-decode-only DTS is left to verify reclassification + if (!dtsInfo.HasNonMonotonicDts) + { + return TimestampRepair.NotApplicable; + } + + // Rewrite timestamps losslessly to a temp file + string tempName = Path.ChangeExtension(FileInfo.FullName, ".tmp14"); + Debug.Assert(FileInfo.FullName != tempName); + + // Decision to modify the media, log once at Warning before the rewrite so it shows at Warning level + Log.Warning("Repairing non-monotonic DTS timestamps : {FileName}", FileInfo.FullName); + if (!Tools.FfMpeg.SetTimestamps(FileInfo.FullName, tempName)) + { + File.Delete(tempName); + return TimestampRepair.Failed; + } + + // Reject unless the payload is byte-identical and the result verifies clean + if ( + !TimestampRepairRegressionGate(FileInfo.FullName, tempName) + || VerifyMediaStreams(new FileInfo(tempName)) != VerifyResult.Clean + ) + { + File.Delete(tempName); + return TimestampRepair.Failed; + } + + // Replace the original with the repaired file + File.Move(tempName, FileInfo.FullName, true); + Log.Information("Timestamp repair succeeded : {FileName}", FileInfo.FullName); + return TimestampRepair.Repaired; + } + + private static bool TimestampRepairRegressionGate(string original, string repaired) => + // Lossless requires every stream's coded payload to be byte-identical before and after + // The streamhash muxer hashes packet data only, so a matching hash proves only timestamps changed + Tools.FfMpeg.GetStreamHashes(original, out Dictionary before) + && Tools.FfMpeg.GetStreamHashes(repaired, out Dictionary after) + && before.Count == after.Count + && before.All(kvp => after.TryGetValue(kvp.Key, out string? hash) && hash == kvp.Value); + public bool SetLastWriteTimeUtc(DateTime lastWriteTimeUtc) { // Conditional @@ -2333,7 +2461,14 @@ public bool TestMediaProps() return true; } - public bool GetBitrateInfo(out BitrateInfo? bitrateInfo) + public bool GetBitrateInfo(out BitrateInfo? bitrateInfo) => + GetPacketAnalysis(Program.Options.QuickScan, out bitrateInfo, out _); + + public bool GetPacketAnalysis( + bool quickScan, + out BitrateInfo? bitrateInfo, + out DtsInfo? dtsInfo + ) { // Use the default track, else the first track VideoProps? videoProps = FfProbeProps.Video.Find(item => @@ -2345,22 +2480,25 @@ public bool GetBitrateInfo(out BitrateInfo? bitrateInfo) ); audioProps ??= FfProbeProps.Audio.FirstOrDefault(); - // Add all packets + // Read all packets once, computing the bitrate and the DTS monotonicity in a single pass bitrateInfo = null; + dtsInfo = null; BitrateInfo packetBitrate = new( videoProps?.Id ?? -1, audioProps?.Id ?? -1, Program.Config.VerifyOptions.MaximumBitrate / 8 ); + DtsInfo packetDts = new(); if ( - !Tools.FfProbe.GetBitratePackets( + !Tools.FfProbe.GetAnalysisPackets( FileInfo.FullName, packet => { - // Convert from void to bool return packetBitrate.Add(packet); + packetDts.Add(packet); return true; - } + }, + quickScan ) ) { @@ -2370,6 +2508,7 @@ public bool GetBitrateInfo(out BitrateInfo? bitrateInfo) // Calculate bitrate packetBitrate.Calculate(); bitrateInfo = packetBitrate; + dtsInfo = packetDts; return true; } diff --git a/PlexCleaner/Program.cs b/PlexCleaner/Program.cs index e2ef29c8..ede7475b 100644 --- a/PlexCleaner/Program.cs +++ b/PlexCleaner/Program.cs @@ -13,12 +13,13 @@ public static class Program // Exit code for an OS-signal interruption (128 + signal number), else 0; set only by PosixSignalHandler private static volatile int s_signalExitCode; - // Serilog to Microsoft.Extensions.Logging bridge shared with library loggers; lives for the - // process lifetime and is disposed at shutdown alongside the logger + // Serilog to Microsoft.Extensions.Logging bridge shared with library loggers private static Microsoft.Extensions.Logging.ILoggerFactory? s_libraryLoggerFactory; public static readonly TimeSpan SnippetTimeSpan = TimeSpan.FromSeconds(30); public static readonly TimeSpan QuickScanTimeSpan = TimeSpan.FromMinutes(3); + public const int QuickScanFrameCount = 1000; + public static CommandLineOptions Options { get; set; } = null!; public static ConfigFileJsonSchema Config { get; set; } = null!; diff --git a/PlexCleaner/VerifyClassifier.cs b/PlexCleaner/VerifyClassifier.cs new file mode 100644 index 00000000..19c28a3d --- /dev/null +++ b/PlexCleaner/VerifyClassifier.cs @@ -0,0 +1,60 @@ +namespace PlexCleaner; + +internal static class VerifyClassifier +{ + // Non-monotonic-DTS muxer warnings emitted by the -f null muxer at error loglevel + private static readonly List s_timestampSignatures = + [ + "non monotonically increasing dts to muxer", + ]; + + public static VerifyResult Classify(string stderr) + { + Accumulator accumulator = new(); + if (!string.IsNullOrEmpty(stderr)) + { + foreach (string line in stderr.Split('\n')) + { + accumulator.Add(line); + } + } + return accumulator.Result; + } + + public sealed class Accumulator + { + private bool _decodeError; + private bool _timestamp; + + public string? FirstError { get; private set; } + + public VerifyResult Result => + _decodeError ? VerifyResult.DecodeError + : _timestamp ? VerifyResult.TimestampOnly + : VerifyResult.Clean; + + public void Add(string line) + { + line = line.Trim(); + if (line.Length == 0) + { + return; + } + + // A benign muxer timestamp warning does not by itself fail verify + if ( + s_timestampSignatures.Any(sig => + line.Contains(sig, StringComparison.OrdinalIgnoreCase) + ) + ) + { + _timestamp = true; + return; + } + + // Anything else at error loglevel is treated as decode corruption, fail closed + _decodeError = true; + FirstError ??= line; + } + } +} diff --git a/PlexCleaner/VerifyResult.cs b/PlexCleaner/VerifyResult.cs new file mode 100644 index 00000000..efa193fe --- /dev/null +++ b/PlexCleaner/VerifyResult.cs @@ -0,0 +1,13 @@ +namespace PlexCleaner; + +public enum VerifyResult +{ + // Decode succeeded with no diagnostic output + Clean, + + // The only diagnostics are muxer timestamp warnings (non-monotonic DTS) + TimestampOnly, + + // A genuine decode or demux corruption signature, or any unrecognized diagnostic, verify fails + DecodeError, +} diff --git a/PlexCleanerTests/ClosedCaptionsProbeTests.cs b/PlexCleanerTests/ClosedCaptionsProbeTests.cs new file mode 100644 index 00000000..c2a8b09e --- /dev/null +++ b/PlexCleanerTests/ClosedCaptionsProbeTests.cs @@ -0,0 +1,49 @@ +using AwesomeAssertions; +using PlexCleaner; +using Xunit; + +namespace PlexCleanerTests; + +public class ClosedCaptionsProbeTests +{ + [Fact] + public void FromJson_ClosedCaptionsPresent_Parsed() + { + // lang=json + const string json = """ + { "programs": [], "stream_groups": [], "streams": [ { "closed_captions": 1 } ] } + """; + FfMpegToolJsonSchema.ClosedCaptionsProbe probe = + FfMpegToolJsonSchema.ClosedCaptionsProbe.FromJson(json); + + _ = probe.Streams.Should().ContainSingle(); + _ = probe.Streams[0].ClosedCaptions.Should().Be(1); + } + + [Fact] + public void FromJson_NoClosedCaptions_Parsed() + { + // lang=json + const string json = """ + { "streams": [ { "closed_captions": 0 } ] } + """; + FfMpegToolJsonSchema.ClosedCaptionsProbe probe = + FfMpegToolJsonSchema.ClosedCaptionsProbe.FromJson(json); + + _ = probe.Streams.Should().ContainSingle(); + _ = probe.Streams[0].ClosedCaptions.Should().Be(0); + } + + [Fact] + public void FromJson_NoStreams_Empty() + { + // lang=json + const string json = """ + { "streams": [] } + """; + FfMpegToolJsonSchema.ClosedCaptionsProbe probe = + FfMpegToolJsonSchema.ClosedCaptionsProbe.FromJson(json); + + _ = probe.Streams.Should().BeEmpty(); + } +} diff --git a/PlexCleanerTests/DtsInfoTests.cs b/PlexCleanerTests/DtsInfoTests.cs new file mode 100644 index 00000000..650aaae5 --- /dev/null +++ b/PlexCleanerTests/DtsInfoTests.cs @@ -0,0 +1,61 @@ +using AwesomeAssertions; +using PlexCleaner; +using Xunit; + +namespace PlexCleanerTests; + +public class DtsInfoTests +{ + private static FfMpegToolJsonSchema.Packet Packet(long streamIndex, double dtsTime) => + new() { StreamIndex = streamIndex, DtsTime = dtsTime }; + + [Fact] + public void Add_MonotonicDts_NoDetection() + { + DtsInfo dtsInfo = new(); + foreach (double dts in new[] { 0.0, 0.04, 0.08, 0.12 }) + { + dtsInfo.Add(Packet(1, dts)); + } + + _ = dtsInfo.HasNonMonotonicDts.Should().BeFalse(); + _ = dtsInfo.NonMonotonicByStream.Should().BeEmpty(); + } + + [Fact] + public void Add_DuplicateDts_Detected() + { + // A repeated DTS (X >= X) is the common benign case + DtsInfo dtsInfo = new(); + dtsInfo.Add(Packet(1, 0.08)); + dtsInfo.Add(Packet(1, 0.08)); + + _ = dtsInfo.HasNonMonotonicDts.Should().BeTrue(); + _ = dtsInfo.NonMonotonicByStream[1].Should().Be(1); + } + + [Fact] + public void Add_BackwardDts_DetectedPerStream() + { + DtsInfo dtsInfo = new(); + dtsInfo.Add(Packet(1, 0.10)); + dtsInfo.Add(Packet(1, 0.05)); + dtsInfo.Add(Packet(2, 0.00)); + dtsInfo.Add(Packet(2, 0.04)); + + _ = dtsInfo.HasNonMonotonicDts.Should().BeTrue(); + _ = dtsInfo.NonMonotonicByStream.Should().ContainKey(1); + _ = dtsInfo.NonMonotonicByStream.Should().NotContainKey(2); + } + + [Fact] + public void Add_NanDts_Ignored() + { + // Packets with no DTS or PTS are skipped, not counted as breaks + DtsInfo dtsInfo = new(); + dtsInfo.Add(Packet(1, double.NaN)); + dtsInfo.Add(Packet(1, double.NaN)); + + _ = dtsInfo.HasNonMonotonicDts.Should().BeFalse(); + } +} diff --git a/PlexCleanerTests/FileNameEscapingTests.cs b/PlexCleanerTests/FileNameEscapingTests.cs deleted file mode 100644 index 26c3eee3..00000000 --- a/PlexCleanerTests/FileNameEscapingTests.cs +++ /dev/null @@ -1,25 +0,0 @@ -using PlexCleaner; -using Xunit; - -namespace PlexCleanerTests; - -public class FileNameEscapingTests -{ - [Theory] - [InlineData(@"\", @"/")] - [InlineData(@":", @"\\:")] - [InlineData(@"'", @"\\\'")] - [InlineData(@",", @"\\\,")] - [InlineData(@";", @"\\\;")] - [InlineData(@"[", @"\\\[")] - [InlineData(@"]", @"\\\]")] - [InlineData( - @"D:\Test\Naming - movie=,.;{}[out0+subcc] (1234) {abc-123} [aaa][bbb][ccc]-def.mkv", - @"D\\:/Test/Naming - movie=\\\,.\\\;{}\\\[out0+subcc\\\] (1234) {abc-123} \\\[aaa\\\]\\\[bbb\\\]\\\[ccc\\\]-def.mkv" - )] - public void Escape_Movie_fileName(string fileName, string escapedName) - { - string escapedFileName = FfProbe.EscapeMovieFileName(fileName); - Assert.Equal(escapedName, escapedFileName); - } -} diff --git a/PlexCleanerTests/PlexCleanerTests.csproj b/PlexCleanerTests/PlexCleanerTests.csproj index 0c986898..47f77ca5 100644 --- a/PlexCleanerTests/PlexCleanerTests.csproj +++ b/PlexCleanerTests/PlexCleanerTests.csproj @@ -18,7 +18,8 @@ - + + diff --git a/PlexCleanerTests/PluginLoaderTests.cs b/PlexCleanerTests/PluginLoaderTests.cs index 6b1fd3a1..b19201d8 100644 --- a/PlexCleanerTests/PluginLoaderTests.cs +++ b/PlexCleanerTests/PluginLoaderTests.cs @@ -22,6 +22,19 @@ public void Load_ExamplePlugin_ReturnsInitializedPlugin() _ = loaded.GetType().Assembly.GetName().Name.Should().Be("MatroskaHeaderCleanup"); } + [Fact] + public void Load_DtsTimestampRepairPlugin_ReturnsInitializedPlugin() + { + IProcessPlugin? plugin = PluginLoader.Load( + new FileInfo(Path.Combine(AppContext.BaseDirectory, "DtsTimestampRepair.dll")) + ); + + _ = plugin.Should().NotBeNull(); + IProcessPlugin loaded = plugin; + _ = loaded.Name.Should().Be("DtsTimestampRepair"); + _ = loaded.GetType().Assembly.GetName().Name.Should().Be("DtsTimestampRepair"); + } + [Fact] public void Load_MissingAssembly_ReturnsNull() { diff --git a/PlexCleanerTests/VerifyClassifierTests.cs b/PlexCleanerTests/VerifyClassifierTests.cs new file mode 100644 index 00000000..6b00e7d2 --- /dev/null +++ b/PlexCleanerTests/VerifyClassifierTests.cs @@ -0,0 +1,69 @@ +using AwesomeAssertions; +using PlexCleaner; +using Xunit; + +namespace PlexCleanerTests; + +public class VerifyClassifierTests +{ + [Fact] + public void Classify_EmptyStderr_ReturnsClean() + { + _ = VerifyClassifier.Classify(string.Empty).Should().Be(VerifyResult.Clean); + _ = VerifyClassifier.Classify(" \n ").Should().Be(VerifyResult.Clean); + } + + [Fact] + public void Classify_OnlyDtsMuxerWarnings_ReturnsTimestampOnly() + { + string stderr = + "[null @ 0x1] Application provided invalid, non monotonically increasing dts to muxer in stream 1: 8 >= 8\n" + + "[null @ 0x1] Application provided invalid, non monotonically increasing dts to muxer in stream 1: 12 >= 12\n"; + _ = VerifyClassifier.Classify(stderr).Should().Be(VerifyResult.TimestampOnly); + } + + [Theory] + [InlineData("[matroska @ 0x1] Invalid data found when processing input")] + [InlineData("[h264 @ 0x1] error while decoding MB 10 20")] + [InlineData("[NULL @ 0x1] Invalid NAL unit size (-1148261185 > 8772).")] + [InlineData("[h264 @ 0x1] mmco: unref short failure")] + [InlineData("[aac @ 0x1] env_facs_q 255 is invalid")] + [InlineData("[matroska,webm @ 0x1] Length 7 indicated by an EBML number exceeds max length 4.")] + public void Classify_DecodeSignature_ReturnsDecodeError(string stderr) => + VerifyClassifier.Classify(stderr).Should().Be(VerifyResult.DecodeError); + + [Fact] + public void Classify_DecodeErrorMixedWithDtsWarning_ReturnsDecodeError() + { + // A decode error co-occurring with benign DTS warnings must still fail + string stderr = + "[null @ 0x1] Application provided invalid, non monotonically increasing dts to muxer in stream 1: 8 >= 8\n" + + "[h264 @ 0x1] error while decoding MB 3 4\n"; + _ = VerifyClassifier.Classify(stderr).Should().Be(VerifyResult.DecodeError); + } + + [Fact] + public void Classify_UnrecognizedStderr_ReturnsDecodeError() => + // Fail closed on anything not explicitly benign + VerifyClassifier + .Classify("[mystery @ 0x1] some unexpected diagnostic line") + .Should() + .Be(VerifyResult.DecodeError); + + [Fact] + public void Accumulator_StreamedLines_ClassifiesAndKeepsFirstError() + { + // The accumulator sees lines one at a time without buffering the whole stderr + VerifyClassifier.Accumulator accumulator = new(); + accumulator.Add( + "[null @ 0x1] Application provided invalid, non monotonically increasing dts to muxer in stream 1: 8 >= 8" + ); + _ = accumulator.Result.Should().Be(VerifyResult.TimestampOnly); + + accumulator.Add("[h264 @ 0x1] error while decoding MB 3 4"); + accumulator.Add("[h264 @ 0x1] Invalid data found"); + + _ = accumulator.Result.Should().Be(VerifyResult.DecodeError); + _ = accumulator.FirstError.Should().Be("[h264 @ 0x1] error while decoding MB 3 4"); + } +} diff --git a/Plugins/DtsTimestampRepair/DtsTimestampRepair.csproj b/Plugins/DtsTimestampRepair/DtsTimestampRepair.csproj new file mode 100644 index 00000000..9ea4b34e --- /dev/null +++ b/Plugins/DtsTimestampRepair/DtsTimestampRepair.csproj @@ -0,0 +1,10 @@ + + + PlexCleaner.Plugins.DtsTimestampRepair + DtsTimestampRepair + + + + + + diff --git a/Plugins/DtsTimestampRepair/DtsTimestampRepairPlugin.cs b/Plugins/DtsTimestampRepair/DtsTimestampRepairPlugin.cs new file mode 100644 index 00000000..068cba57 --- /dev/null +++ b/Plugins/DtsTimestampRepair/DtsTimestampRepairPlugin.cs @@ -0,0 +1,90 @@ +using Serilog; + +namespace PlexCleaner.Plugins.DtsTimestampRepair; + +// Retroactively repair files that an older PlexCleaner version marked RepairFailed because verify +// wrongly rejected a benign non-monotonic-DTS muxer warning. Re-verifies each RepairFailed file, clears +// the flag when the only problem is timestamps, and losslessly rewrites the timestamps (setts) when the +// DTS is demux-visible. Reuses PlexCleaner.ProcessFile.RepairTimestamps. +public sealed class DtsTimestampRepairPlugin : IProcessPlugin +{ + private ILogger _logger = Log.Logger; + + // The PlexCleaner version whose public API this plugin was built and tested against. PluginApi.Version + // only guards the plugin contract; the ProcessFile methods this plugin calls can change in any release, + // so pin against the tested application version as well. + private const string TestedApplicationVersion = "3.21"; + + public string Name => "DtsTimestampRepair"; + + public bool Initialize(IPluginHost host) + { + _logger = host.Logger; + + // Refuse to run against an incompatible plugin contract + if (host.PluginApiVersion != PluginApi.Version) + { + _logger.Error( + "Incompatible plugin API version : host {HostVersion} != plugin {PluginVersion}", + host.PluginApiVersion, + PluginApi.Version + ); + return false; + } + + // Warn when the running PlexCleaner version differs from the tested version, since the public API + // this plugin calls could have changed + if ( + !MajorMinor(host.ApplicationVersion) + .Equals(TestedApplicationVersion, StringComparison.Ordinal) + ) + { + _logger.Warning( + "Plugin tested against PlexCleaner {TestedVersion} but host is {HostVersion}, internal APIs may differ", + TestedApplicationVersion, + host.ApplicationVersion + ); + } + + _logger.Information( + "{Name} initialized : {AppVersion} : {Os}", + Name, + host.ApplicationVersion, + host.OperatingSystem + ); + return true; + } + + // Reduce a version string like "3.21.1.0" to "3.21" to compare against the tested major.minor + private static string MajorMinor(string version) + { + string[] parts = version.Split('.'); + return parts.Length >= 2 ? $"{parts[0]}.{parts[1]}" : version; + } + + public bool ProcessFile(string fileName) + { + // The driver passes every file, skip anything that is not a Matroska file + if (!SidecarFile.IsMkvFile(fileName)) + { + return true; + } + + ProcessFile processFile = new(fileName); + if (!processFile.GetMediaProps()) + { + return false; + } + + // Only revisit files a previous run gave up on + if (!processFile.State.HasFlag(SidecarFile.StatesType.RepairFailed)) + { + return true; + } + + // Re-verify and clear the RepairFailed flag when the failure was a benign timestamp issue, + // losslessly repairing the timestamps when possible, RepairTimestamps keeps the sidecar in sync + bool modified = false; + return processFile.RepairTimestamps(ref modified); + } +} diff --git a/README.md b/README.md index ed231bef..eedc6bdc 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,14 @@ Utility to optimize media files for Direct Play in Plex, Emby, Jellyfin, etc. ### Release Notes +**Version: 3.21**: + +**Summary:** + +- Repair non-monotonic DTS losslessly with the `setts` bitstream filter instead of failing repair permanently; a non-monotonic DTS is a correctable verify failure, not decode corruption. +- Switched closed caption detection to `ffprobe -analyze_frames`, and consolidated the bitrate and DTS packet analyses into a single packet pass. +- Added the `DtsTimestampRepair` example plugin that revisits files a previous version marked `RepairFailed` and clears or losslessly repairs benign timestamp failures. + **Version: 3.20**: **Summary:** @@ -149,6 +157,7 @@ Common examples of issues resolved by the `process` command: **Performance & Integrity:** - Corrupt media streams → Verify integrity and attempt automatic repair. +- Non-monotonic DTS timestamps → Losslessly rewrite the packet timestamps using `setts`. - Matroska files that fail player Direct Play despite passing tool checks → Detect an unusable seek index (SeekHead/Cues) and re-multiplex. - High bitrate content → Warn when exceeding network capacity (WiFi/100Mbps Ethernet). @@ -851,7 +860,10 @@ public interface IProcessPlugin } ``` -`Initialize` receives an `IPluginHost` with the deterministic `PluginApiVersion`, the application and OS versions, and a `Serilog.ILogger` to log through. `ProcessFile` reuses the public processing API, for example `new ProcessFile(fileName)` then `RepairMatroskaStructure(...)`. See the [`MatroskaHeaderCleanup`](./Plugins/MatroskaHeaderCleanup/) example, which re-checks and repairs the Matroska seek-index structure on already-verified files. +`Initialize` receives an `IPluginHost` with the deterministic `PluginApiVersion`, the application and OS versions, and a `Serilog.ILogger` to log through. `ProcessFile` reuses the public processing API, for example `new ProcessFile(fileName)` then `RepairMatroskaStructure(...)`. Two examples are included: + +- [`MatroskaHeaderCleanup`](./Plugins/MatroskaHeaderCleanup/) re-checks and repairs the Matroska seek-index structure on already-verified files. +- [`DtsTimestampRepair`](./Plugins/DtsTimestampRepair/) revisits files that an older version marked `RepairFailed`, re-verifies them, clears the flag when the only problem was a benign non-monotonic DTS, and losslessly repairs the timestamps (`setts`) when the DTS is demux-visible. Notes: diff --git a/cspell.json b/cspell.json index b74647c8..f6a0aa13 100644 --- a/cspell.json +++ b/cspell.json @@ -43,6 +43,7 @@ "deinterlace", "deinterlaced", "deinterlacing", + "demux", "derbend", "devel", "dockerhub", @@ -152,6 +153,8 @@ "numfmt", "NVENC", "Oughta", + "OUTDTS", + "OUTPTS", "parameterless", "partitioner", "piete", @@ -192,9 +195,11 @@ "Serilog", "setparams", "settingsfile", + "setts", "SMPTE", "snupkg", "softprops", + "streamhash", "stylecop", "subcc", "subdir", diff --git a/version.json b/version.json index 1947a534..8c723482 100644 --- a/version.json +++ b/version.json @@ -1,6 +1,6 @@ { "$schema": "https://raw.githubusercontent.com/dotnet/Nerdbank.GitVersioning/master/src/NerdBank.GitVersioning/version.schema.json", - "version": "3.20", + "version": "3.21", "publicReleaseRefSpec": [ "^refs/heads/main$" ], From 5f5f1091ebbb0ea189c8e6c4d14e37e30dd4a0b6 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Mon, 13 Jul 2026 13:38:05 -0700 Subject: [PATCH 03/19] Keep a detected unrepairable DTS reported as RepairFailed (#834) A non-monotonic DTS that verify detects but the lossless setts repair cannot fix now stays RepairFailed instead of being cleared to Verified. Only a successful lossless repair or a clean re-verify clears the flag. Follow-up to #833. --- HISTORY.md | 2 +- PlexCleaner/ProcessFile.cs | 88 +++++++------------ PlexCleaner/VerifyClassifier.cs | 2 +- .../DtsTimestampRepairPlugin.cs | 11 ++- README.md | 6 +- 5 files changed, 44 insertions(+), 65 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index 5a9feb97..22a9eccc 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -8,7 +8,7 @@ Utility to optimize media files for Direct Play in Plex, Emby, Jellyfin, etc. - Repair non-monotonic DTS muxer warnings losslessly instead of failing repair permanently. - `ffmpeg -f null` can exit `0` yet emit `Application provided invalid, non monotonically increasing dts to muxer` for files that may decode and play correctly. - The previous "any stderr means failure" rule promoted this muxer-interleaving artifact to a hard `VerifyFailed`/`RepairFailed`, and a re-encode could not fix it because Matroska stores no DTS and ffmpeg re-derives a non-monotonic timeline on read. - - Verify now classifies the decode diagnostics deterministically as clean, a benign timestamp-only failure, or a decode error; the timestamp-only failure is correctable rather than permanent, and everything else fails (fail-closed, so an unrecognized diagnostic fails as a decode error). + - Verify now classifies the decode diagnostics deterministically as clean, a timestamp-only failure, or a decode error; a timestamp-only failure is repaired losslessly when the break is demux-visible and otherwise stays reported, and everything else fails (fail-closed, so an unrecognized diagnostic fails as a decode error). - The classification streams the output line by line, so memory stays bounded even when a file emits a warning per packet ([#827](https://github.com/ptr727/PlexCleaner/issues/827)). - Added a lossless timestamp repair as the first repair tier. - When verification detects a demux-visible non-monotonic DTS, the audio packet timestamps are rewritten to be strictly monotonic using the `setts` bitstream filter with a stream copy (no re-encode), then re-verified. diff --git a/PlexCleaner/ProcessFile.cs b/PlexCleaner/ProcessFile.cs index 0f286739..53326ebf 100644 --- a/PlexCleaner/ProcessFile.cs +++ b/PlexCleaner/ProcessFile.cs @@ -2174,7 +2174,7 @@ public bool RepairTimestamps(ref bool modified) _sidecarFile.State &= ~SidecarFile.StatesType.RepairFailed; return Refresh(false); case VerifyResult.TimestampOnly: - // Benign non-monotonic DTS, clean losslessly when demux-visible + // Detected non-monotonic DTS, repair losslessly when demux-visible, else stays reported return RepairTimestampsAndSetState(ref modified); case VerifyResult.DecodeError: // Genuine decode corruption, not repairable here, leave the state unchanged @@ -2184,65 +2184,45 @@ public bool RepairTimestamps(ref bool modified) } } - private enum TimestampRepair - { - // No demux-visible break, the muxer warning is a post-decode artifact with no defect to repair - NotApplicable, - - // Timestamps rewritten losslessly - Repaired, - - // A demux-visible break, but the lossless repair could not complete - Failed, - } - private bool RepairTimestampsAndSetState(ref bool modified) { - switch (TryLosslessTimestampRepair()) + // A detected non-monotonic DTS is a failure, repair it losslessly when the break is demux-visible + if (TryLosslessTimestampRepair()) { - case TimestampRepair.Repaired: - _sidecarFile.State |= SidecarFile.StatesType.Verified; - _sidecarFile.State &= ~SidecarFile.StatesType.VerifyFailed; - _sidecarFile.State |= SidecarFile.StatesType.Repaired; - _sidecarFile.State &= ~SidecarFile.StatesType.RepairFailed; - modified = true; - return Refresh(true); - case TimestampRepair.NotApplicable: - // No demux-visible defect, the media is benign, mark verified without a rewrite - _sidecarFile.State |= SidecarFile.StatesType.Verified; - _sidecarFile.State &= ~SidecarFile.StatesType.VerifyFailed; - _sidecarFile.State &= ~SidecarFile.StatesType.RepairFailed; - return Refresh(false); - case TimestampRepair.Failed: - // Do not touch state on cancellation, the caller retries next run - if (Program.IsCancelled()) - { - return false; - } - // The lossless repair failed, leave it as a repair failure - _sidecarFile.State |= SidecarFile.StatesType.VerifyFailed; - _sidecarFile.State &= ~SidecarFile.StatesType.Verified; - _sidecarFile.State |= SidecarFile.StatesType.RepairFailed; - _sidecarFile.State &= ~SidecarFile.StatesType.Repaired; - _ = Refresh(false); - return false; - default: - throw new NotImplementedException(); + _sidecarFile.State |= SidecarFile.StatesType.Verified; + _sidecarFile.State &= ~SidecarFile.StatesType.VerifyFailed; + _sidecarFile.State |= SidecarFile.StatesType.Repaired; + _sidecarFile.State &= ~SidecarFile.StatesType.RepairFailed; + modified = true; + return Refresh(true); } - } - private TimestampRepair TryLosslessTimestampRepair() - { - // Could not analyze packets, treat as a repair failure rather than assume benign - if (!GetPacketAnalysis(false, out _, out DtsInfo? dtsInfo) || dtsInfo == null) + // Do not touch state on cancellation, the caller retries next run + if (Program.IsCancelled()) { - return TimestampRepair.Failed; + return false; } - // No demux-visible break, a post-decode-only DTS is left to verify reclassification - if (!dtsInfo.HasNonMonotonicDts) + // A detected DTS we could not repair stays reported as a failure, a detected issue is not cleared + _sidecarFile.State |= SidecarFile.StatesType.VerifyFailed; + _sidecarFile.State &= ~SidecarFile.StatesType.Verified; + _sidecarFile.State |= SidecarFile.StatesType.RepairFailed; + _sidecarFile.State &= ~SidecarFile.StatesType.Repaired; + _ = Refresh(false); + return false; + } + + private bool TryLosslessTimestampRepair() + { + // Only a demux-visible non-monotonic DTS can be rewritten losslessly, a post-decode-only break has + // no demux target and stays a reported failure, an analysis failure is treated as unrepaired too + if ( + !GetPacketAnalysis(false, out _, out DtsInfo? dtsInfo) + || dtsInfo == null + || !dtsInfo.HasNonMonotonicDts + ) { - return TimestampRepair.NotApplicable; + return false; } // Rewrite timestamps losslessly to a temp file @@ -2254,7 +2234,7 @@ private TimestampRepair TryLosslessTimestampRepair() if (!Tools.FfMpeg.SetTimestamps(FileInfo.FullName, tempName)) { File.Delete(tempName); - return TimestampRepair.Failed; + return false; } // Reject unless the payload is byte-identical and the result verifies clean @@ -2264,13 +2244,13 @@ private TimestampRepair TryLosslessTimestampRepair() ) { File.Delete(tempName); - return TimestampRepair.Failed; + return false; } // Replace the original with the repaired file File.Move(tempName, FileInfo.FullName, true); Log.Information("Timestamp repair succeeded : {FileName}", FileInfo.FullName); - return TimestampRepair.Repaired; + return true; } private static bool TimestampRepairRegressionGate(string original, string repaired) => diff --git a/PlexCleaner/VerifyClassifier.cs b/PlexCleaner/VerifyClassifier.cs index 19c28a3d..a6a158ad 100644 --- a/PlexCleaner/VerifyClassifier.cs +++ b/PlexCleaner/VerifyClassifier.cs @@ -41,7 +41,7 @@ public void Add(string line) return; } - // A benign muxer timestamp warning does not by itself fail verify + // A muxer timestamp warning classifies as TimestampOnly, distinct from a decode error if ( s_timestampSignatures.Any(sig => line.Contains(sig, StringComparison.OrdinalIgnoreCase) diff --git a/Plugins/DtsTimestampRepair/DtsTimestampRepairPlugin.cs b/Plugins/DtsTimestampRepair/DtsTimestampRepairPlugin.cs index 068cba57..08c45279 100644 --- a/Plugins/DtsTimestampRepair/DtsTimestampRepairPlugin.cs +++ b/Plugins/DtsTimestampRepair/DtsTimestampRepairPlugin.cs @@ -2,10 +2,9 @@ namespace PlexCleaner.Plugins.DtsTimestampRepair; -// Retroactively repair files that an older PlexCleaner version marked RepairFailed because verify -// wrongly rejected a benign non-monotonic-DTS muxer warning. Re-verifies each RepairFailed file, clears -// the flag when the only problem is timestamps, and losslessly rewrites the timestamps (setts) when the -// DTS is demux-visible. Reuses PlexCleaner.ProcessFile.RepairTimestamps. +// Revisit files a previous run marked RepairFailed and losslessly repair a demux-visible non-monotonic +// DTS with the setts filter, clearing the flag only on a successful repair or a clean re-verify. A +// detected DTS that cannot be repaired stays RepairFailed. Reuses PlexCleaner.ProcessFile.RepairTimestamps. public sealed class DtsTimestampRepairPlugin : IProcessPlugin { private ILogger _logger = Log.Logger; @@ -82,8 +81,8 @@ public bool ProcessFile(string fileName) return true; } - // Re-verify and clear the RepairFailed flag when the failure was a benign timestamp issue, - // losslessly repairing the timestamps when possible, RepairTimestamps keeps the sidecar in sync + // Re-verify and losslessly repair a demux-visible DTS. Clear RepairFailed only on success. + // An unrepairable DTS stays reported. RepairTimestamps keeps the sidecar in sync. bool modified = false; return processFile.RepairTimestamps(ref modified); } diff --git a/README.md b/README.md index eedc6bdc..728ddc14 100644 --- a/README.md +++ b/README.md @@ -27,9 +27,9 @@ Utility to optimize media files for Direct Play in Plex, Emby, Jellyfin, etc. **Summary:** -- Repair non-monotonic DTS losslessly with the `setts` bitstream filter instead of failing repair permanently; a non-monotonic DTS is a correctable verify failure, not decode corruption. +- Treat a non-monotonic DTS as a verify failure, not decode corruption. Repair it losslessly with the `setts` bitstream filter when the break is demux-visible, otherwise keep it reported as a failure. - Switched closed caption detection to `ffprobe -analyze_frames`, and consolidated the bitrate and DTS packet analyses into a single packet pass. -- Added the `DtsTimestampRepair` example plugin that revisits files a previous version marked `RepairFailed` and clears or losslessly repairs benign timestamp failures. +- Added the `DtsTimestampRepair` example plugin that revisits files a previous version marked `RepairFailed` and losslessly repairs a demux-visible non-monotonic DTS, clearing the flag on success. **Version: 3.20**: @@ -863,7 +863,7 @@ public interface IProcessPlugin `Initialize` receives an `IPluginHost` with the deterministic `PluginApiVersion`, the application and OS versions, and a `Serilog.ILogger` to log through. `ProcessFile` reuses the public processing API, for example `new ProcessFile(fileName)` then `RepairMatroskaStructure(...)`. Two examples are included: - [`MatroskaHeaderCleanup`](./Plugins/MatroskaHeaderCleanup/) re-checks and repairs the Matroska seek-index structure on already-verified files. -- [`DtsTimestampRepair`](./Plugins/DtsTimestampRepair/) revisits files that an older version marked `RepairFailed`, re-verifies them, clears the flag when the only problem was a benign non-monotonic DTS, and losslessly repairs the timestamps (`setts`) when the DTS is demux-visible. +- [`DtsTimestampRepair`](./Plugins/DtsTimestampRepair/) revisits files that an older version marked `RepairFailed`, re-verifies them, and losslessly repairs the timestamps (`setts`) when the non-monotonic DTS is demux-visible, clearing the flag on success; a detected DTS it cannot repair stays reported. Notes: From 9137b2be22e5b1756145b21d9192ccde786c3575 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Mon, 13 Jul 2026 15:18:51 -0700 Subject: [PATCH 04/19] Ignore ffmpeg repeat markers and log unique verify errors (#835) ffmpeg's 'Last message repeated N times' line was misclassified as a decode error, so a duplicate-DTS file was wrongly failed instead of losslessly repaired. Ignore those markers. Also log the unique decode-error lines (deduped by a normalized key, capped) so a failure reports every distinct error. Follow-up to #833/#834. --- PlexCleaner/FfMpegTool.cs | 4 +- PlexCleaner/VerifyClassifier.cs | 41 ++++++++++++++++---- PlexCleanerTests/VerifyClassifierTests.cs | 47 ++++++++++++++++------- 3 files changed, 69 insertions(+), 23 deletions(-) diff --git a/PlexCleaner/FfMpegTool.cs b/PlexCleaner/FfMpegTool.cs index f0776e93..3d169839 100644 --- a/PlexCleaner/FfMpegTool.cs +++ b/PlexCleaner/FfMpegTool.cs @@ -179,8 +179,8 @@ public VerifyResult VerifyMedia(string fileName) } if (verifyResult == VerifyResult.DecodeError) { - // A silent non-zero exit has no error line, omit the empty field rather than logging blank - string error = CleanForLog(classifier.FirstError ?? string.Empty); + // Log the unique error lines, a silent non-zero exit has none so omit the empty field + string error = CleanForLog(string.Join(" | ", classifier.Errors)); if (string.IsNullOrEmpty(error)) { Log.Error( diff --git a/PlexCleaner/VerifyClassifier.cs b/PlexCleaner/VerifyClassifier.cs index a6a158ad..edd6d411 100644 --- a/PlexCleaner/VerifyClassifier.cs +++ b/PlexCleaner/VerifyClassifier.cs @@ -1,6 +1,8 @@ +using System.Text.RegularExpressions; + namespace PlexCleaner; -internal static class VerifyClassifier +internal static partial class VerifyClassifier { // Non-monotonic-DTS muxer warnings emitted by the -f null muxer at error loglevel private static readonly List s_timestampSignatures = @@ -8,6 +10,13 @@ internal static class VerifyClassifier "non monotonically increasing dts to muxer", ]; + // Mask pointer addresses and standalone numbers so lines differing only by those collapse to one + // key; word boundaries keep digits inside identifiers like h264 or mpeg2 so distinct codecs stay apart + [GeneratedRegex(@"0x[0-9a-fA-F]+|\b[0-9]+\b")] + private static partial Regex VariableDataRegex(); + + private static string NormalizeKey(string line) => VariableDataRegex().Replace(line, "*"); + public static VerifyResult Classify(string stderr) { Accumulator accumulator = new(); @@ -23,13 +32,21 @@ public static VerifyResult Classify(string stderr) public sealed class Accumulator { - private bool _decodeError; + // Backstop against a pathological file with many distinct error types + private const int MaxErrorLines = 50; + private bool _timestamp; - public string? FirstError { get; private set; } + // Unique decode-error lines kept for the failure log, deduped by normalized key, insertion ordered + private readonly HashSet _errorKeys = []; + private readonly List _errors = []; + + public bool HasErrors => _errors.Count > 0; + + public IReadOnlyList Errors => _errors; public VerifyResult Result => - _decodeError ? VerifyResult.DecodeError + HasErrors ? VerifyResult.DecodeError : _timestamp ? VerifyResult.TimestampOnly : VerifyResult.Clean; @@ -41,6 +58,13 @@ public void Add(string line) return; } + // ffmpeg collapses consecutive identical messages into this marker, it repeats the previous + // line's classification which is already recorded, so ignore it + if (line.StartsWith("Last message repeated", StringComparison.Ordinal)) + { + return; + } + // A muxer timestamp warning classifies as TimestampOnly, distinct from a decode error if ( s_timestampSignatures.Any(sig => @@ -52,9 +76,12 @@ public void Add(string line) return; } - // Anything else at error loglevel is treated as decode corruption, fail closed - _decodeError = true; - FirstError ??= line; + // Any other line at error loglevel is a decode error, fail closed; keep the unique lines for + // the log, deduped by normalized key and capped as a backstop + if (_errors.Count < MaxErrorLines && _errorKeys.Add(NormalizeKey(line))) + { + _errors.Add(line); + } } } } diff --git a/PlexCleanerTests/VerifyClassifierTests.cs b/PlexCleanerTests/VerifyClassifierTests.cs index 6b00e7d2..3b679700 100644 --- a/PlexCleanerTests/VerifyClassifierTests.cs +++ b/PlexCleanerTests/VerifyClassifierTests.cs @@ -22,20 +22,27 @@ public void Classify_OnlyDtsMuxerWarnings_ReturnsTimestampOnly() _ = VerifyClassifier.Classify(stderr).Should().Be(VerifyResult.TimestampOnly); } + [Fact] + public void Classify_DtsWarningFollowedByRepeatMarker_ReturnsTimestampOnly() + { + // ffmpeg collapses consecutive identical DTS warnings, the repeat marker must not fail the file + string stderr = + "[null @ 0x1] Application provided invalid, non monotonically increasing dts to muxer in stream 1: 8 >= 8\n" + + " Last message repeated 1 times\n"; + _ = VerifyClassifier.Classify(stderr).Should().Be(VerifyResult.TimestampOnly); + } + [Theory] [InlineData("[matroska @ 0x1] Invalid data found when processing input")] [InlineData("[h264 @ 0x1] error while decoding MB 10 20")] [InlineData("[NULL @ 0x1] Invalid NAL unit size (-1148261185 > 8772).")] - [InlineData("[h264 @ 0x1] mmco: unref short failure")] - [InlineData("[aac @ 0x1] env_facs_q 255 is invalid")] - [InlineData("[matroska,webm @ 0x1] Length 7 indicated by an EBML number exceeds max length 4.")] - public void Classify_DecodeSignature_ReturnsDecodeError(string stderr) => + [InlineData("[truehd @ 0x1] quant_step_size larger than huff_lsbs")] + public void Classify_DecodeError_ReturnsDecodeError(string stderr) => VerifyClassifier.Classify(stderr).Should().Be(VerifyResult.DecodeError); [Fact] public void Classify_DecodeErrorMixedWithDtsWarning_ReturnsDecodeError() { - // A decode error co-occurring with benign DTS warnings must still fail string stderr = "[null @ 0x1] Application provided invalid, non monotonically increasing dts to muxer in stream 1: 8 >= 8\n" + "[h264 @ 0x1] error while decoding MB 3 4\n"; @@ -51,19 +58,31 @@ public void Classify_UnrecognizedStderr_ReturnsDecodeError() => .Be(VerifyResult.DecodeError); [Fact] - public void Accumulator_StreamedLines_ClassifiesAndKeepsFirstError() + public void Accumulator_RepeatedErrorType_DedupedToUniqueLines() { - // The accumulator sees lines one at a time without buffering the whole stderr + // Lines differing only by pointer address or numbers collapse to one entry, distinct types do not VerifyClassifier.Accumulator accumulator = new(); - accumulator.Add( - "[null @ 0x1] Application provided invalid, non monotonically increasing dts to muxer in stream 1: 8 >= 8" - ); - _ = accumulator.Result.Should().Be(VerifyResult.TimestampOnly); + accumulator.Add("[pgssub @ 0x5f38] Unknown subtitle segment type 0x78, length 55981"); + accumulator.Add("[pgssub @ 0x5f38] Unknown subtitle segment type 0x93, length 2183"); + accumulator.Add("[aac @ 0x62] Prediction is not allowed in AAC-LC"); + _ = accumulator.Result.Should().Be(VerifyResult.DecodeError); + _ = accumulator.Errors.Should().HaveCount(2); + _ = accumulator + .Errors[0] + .Should() + .Be("[pgssub @ 0x5f38] Unknown subtitle segment type 0x78, length 55981"); + _ = accumulator.Errors[1].Should().Be("[aac @ 0x62] Prediction is not allowed in AAC-LC"); + } + + [Fact] + public void Accumulator_DifferentCodecSameMessage_NotDeduped() + { + // Digits inside a codec identifier must not be masked, so distinct codecs stay separate + VerifyClassifier.Accumulator accumulator = new(); accumulator.Add("[h264 @ 0x1] error while decoding MB 3 4"); - accumulator.Add("[h264 @ 0x1] Invalid data found"); + accumulator.Add("[h265 @ 0x2] error while decoding MB 3 4"); - _ = accumulator.Result.Should().Be(VerifyResult.DecodeError); - _ = accumulator.FirstError.Should().Be("[h264 @ 0x1] error while decoding MB 3 4"); + _ = accumulator.Errors.Should().HaveCount(2); } } From ef79cd889616d0e837b6c2360b5ad322829aa2ad Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Mon, 13 Jul 2026 19:47:07 -0700 Subject: [PATCH 05/19] Produce full output for the setts timestamp repair (#836) setts is a lossless stream-copy repair but honored --testsnippets, writing a 30s snippet that failed the full-file byte-identical gate. Produce full output so the gate and re-verify validate the whole file. Also: README current-version-only summary + AGENTS note. Follow-up to #835. --- AGENTS.md | 2 +- PlexCleaner/FfMpegTool.cs | 4 +++- README.md | 15 ++------------- 3 files changed, 6 insertions(+), 15 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index ccae2831..58711111 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -177,7 +177,7 @@ Anti-pattern: don't keep flipping the code on the same style point. Flip the rul - **Brownfield analyzer relaxations.** `Directory.Build.props` sets strict analysis; because this is a pre-existing console app, a specific set of analyzer rules are relaxed to suggestion in [`.editorconfig`](./.editorconfig), each documented inline. Prefer fixing new violations over adding relaxations. - **Spell check.** The cspell word list and path exclusions live in [`cspell.json`](./cspell.json), the single source shared by the editor and CI. Do not keep a parallel word list in the `.code-workspace` file. - **Run CI CLI tooling via Docker.** The linters CI uses (actionlint, markdownlint-cli2, shellcheck, cspell, etc.) need not be installed on the host - run them from their official images (e.g. `docker run --rm -v "$PWD:/repo" -w /repo rhysd/actionlint`) to reproduce a CI check locally before pushing. -- **Release notes.** Keep a short summary in [`README.md`](./README.md) and the full history in [`HISTORY.md`](./HISTORY.md); update both when cutting a release. +- **Release notes.** Keep a short summary in [`README.md`](./README.md) and the full history in [`HISTORY.md`](./HISTORY.md); update both when cutting a release. `README.md` carries the summary for the **current version only** - when bumping the version, replace the previous version's summary rather than appending; prior versions live in `HISTORY.md`. ## Communicating with the User diff --git a/PlexCleaner/FfMpegTool.cs b/PlexCleaner/FfMpegTool.cs index 3d169839..241fbc75 100644 --- a/PlexCleaner/FfMpegTool.cs +++ b/PlexCleaner/FfMpegTool.cs @@ -363,9 +363,11 @@ public bool SetTimestamps(string inputName, string outputName) File.Delete(outputName); // Build command line, the escaped comma separates the setts option arguments + // No TestSnippets: this lossless stream-copy repair must produce the full file so the + // byte-identical gate and re-verify validate the whole file, not an unrepresentative snippet Command command = GetBuilder() .GlobalOptions(options => options.Default()) - .InputOptions(options => options.Default().TestSnippets().InputFile(inputName)) + .InputOptions(options => options.Default().InputFile(inputName)) .OutputOptions(options => options .MapAllCodecCopy() diff --git a/README.md b/README.md index 728ddc14..ed4de920 100644 --- a/README.md +++ b/README.md @@ -27,20 +27,9 @@ Utility to optimize media files for Direct Play in Plex, Emby, Jellyfin, etc. **Summary:** -- Treat a non-monotonic DTS as a verify failure, not decode corruption. Repair it losslessly with the `setts` bitstream filter when the break is demux-visible, otherwise keep it reported as a failure. +- Treat non-monotonic DTS errors as a verify failure, and attempt to repair it losslessly with the `setts` bitstream filter. - Switched closed caption detection to `ffprobe -analyze_frames`, and consolidated the bitrate and DTS packet analyses into a single packet pass. -- Added the `DtsTimestampRepair` example plugin that revisits files a previous version marked `RepairFailed` and losslessly repairs a demux-visible non-monotonic DTS, clearing the flag on success. - -**Version: 3.20**: - -**Summary:** - -- Reworked logging: added `--loglevel` (`Verbose` ... `Fatal`) to select the log level, `--logclear` (the log file now appends by default), and `--logelevate` to opt into raising a file's level to `Information` after a warning or error. `--logwarning` and `--logappend` are deprecated. Low-level tool and per-track chatter is now logged at `Debug`/`Verbose`. -- Always log the end-of-run summary, and handle stop signals (`docker stop`, `Ctrl+C`) so processing stops gracefully and the summary and exit code are logged before exit. -- Normalize multiple or redundant `Default` track flags instead of only warning about them. -- Fixed an `idet` interlace-detection defect where ffmpeg emitting its statistics more than once could cause the counts to parse incorrectly and detection to fail; also improved detection reporting (detection source and a self-describing reason). -- Added a `custom` command that runs a user-provided plugin assembly over the media files for bespoke re-processing or repair, see [Custom Plugins](#custom-plugins). -- Fixed closed caption removal for H.265/HEVC video that was incorrectly reported as an unsupported format (HDR10 and HDR10+ HEVC content remains guarded). +- Added the `DtsTimestampRepair` example plugin that attempts non-monotonic DTS repairs on `RepairFailed` files. See [Release History](./HISTORY.md) for complete release notes and older versions. From b25f8c75973e23d542889b6dd96861624ea46803 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Mon, 13 Jul 2026 20:04:53 -0700 Subject: [PATCH 06/19] Restrict --testsnippets to slow encode operations (#837) Fast stream-copy and remux operations no longer honor --testsnippets, so remuxes and the lossless timestamp repair produce full output validated on the whole file. Kept on the re-encode and deinterlace paths. Also fixes a README release-note grammar nit. Follow-up to #836. --- HISTORY.md | 1 + PlexCleaner/CommandLineOptions.cs | 3 ++- PlexCleaner/FfMpegBuilder.cs | 2 ++ PlexCleaner/FfMpegTool.cs | 4 ++-- PlexCleaner/MkvMergeTool.cs | 8 ++++---- README.md | 2 +- 6 files changed, 12 insertions(+), 8 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index 22a9eccc..826bdb0e 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -17,6 +17,7 @@ Utility to optimize media files for Direct Play in Plex, Emby, Jellyfin, etc. - Switched closed caption detection to `ffprobe -analyze_frames -show_entries stream=closed_captions`, replacing the `movie=...[out0+subcc]` lavfi filter and its QuickScan snippet-remux workaround; QuickScan now bounds the scan with `-read_intervals`. - Added the `DtsTimestampRepair` example plugin. - It revisits files that a previous version marked `RepairFailed`, re-verifies them, clears the flag when the only problem was timestamps, and losslessly repairs the timestamps when the DTS is demux-visible. Not available in AOT builds. + - Restricted `--testsnippets` to slow re-encode and deinterlace operations. Fast remux, stream-copy, and the lossless timestamp repair now always produce full output, so a repair or remux is validated on the whole file rather than an unrepresentative leading clip; a snippet had caused the timestamp-repair byte-identical gate to fail during testing. - Version 3.20: - Switched tool downloads and the application version check to the resilient HTTP client in `ptr727.Utilities` (retry with backoff and a circuit breaker via `Microsoft.Extensions.Http.Resilience`), replacing the plain `HttpClient`. - Enabled closed caption removal for H.265/HEVC video: the SEI NAL unit lookup keyed on `h265` never matched FFprobe's `hevc` codec name, so HEVC files were incorrectly reported as an "Unsupported video format for Closed Captions removal". HEVC video (excluding HDR10 and HDR10+ content, which remains guarded) is now cleaned using the `filter_units=remove_types=39` bitstream filter, same as H.264 and MPEG-2. diff --git a/PlexCleaner/CommandLineOptions.cs b/PlexCleaner/CommandLineOptions.cs index 01c122c8..eb2b1d82 100644 --- a/PlexCleaner/CommandLineOptions.cs +++ b/PlexCleaner/CommandLineOptions.cs @@ -116,7 +116,8 @@ private class CommandHandler(Func action) : SynchronousCommand private readonly Option _testSnippetsOption = new("--testsnippets") { - Description = "Create short media file clips", + Description = + "Shorten re-encoded output to a clip to speed up testing (does not affect remux or copy operations)", HelpName = "boolean", }; diff --git a/PlexCleaner/FfMpegBuilder.cs b/PlexCleaner/FfMpegBuilder.cs index b60c3f8b..d421a932 100644 --- a/PlexCleaner/FfMpegBuilder.cs +++ b/PlexCleaner/FfMpegBuilder.cs @@ -89,6 +89,8 @@ public InputOptions SeekStart(TimeSpan timeSpan) => public InputOptions SeekStop(TimeSpan timeSpan) => timeSpan == TimeSpan.Zero ? this : SeekStop().Add($"{(int)timeSpan.TotalSeconds}"); + // Apply only to slow encode operations; fast stream-copy and remux operations produce full output + // so a repair or remux is validated on the whole file, not an unrepresentative snippet public InputOptions TestSnippets() => Program.Options.TestSnippets ? SeekStop(Program.SnippetTimeSpan) : this; diff --git a/PlexCleaner/FfMpegTool.cs b/PlexCleaner/FfMpegTool.cs index 241fbc75..9ec8de06 100644 --- a/PlexCleaner/FfMpegTool.cs +++ b/PlexCleaner/FfMpegTool.cs @@ -213,7 +213,7 @@ public bool ReMuxToFormat(string inputName, string outputName, string format) // Build command line Command command = GetBuilder() .GlobalOptions(options => options.Default()) - .InputOptions(options => options.Default().TestSnippets().InputFile(inputName)) + .InputOptions(options => options.Default().InputFile(inputName)) .OutputOptions(options => options.MapAllCodecCopy().Default().Format(format).OutputFile(outputName) ) @@ -442,7 +442,7 @@ public bool RemoveNalUnits(string inputName, int nalUnit, string outputName) // Build command line Command command = GetBuilder() .GlobalOptions(options => options.Default()) - .InputOptions(options => options.Default().TestSnippets().InputFile(inputName)) + .InputOptions(options => options.Default().InputFile(inputName)) .OutputOptions(options => options .MapAllCodecCopy() diff --git a/PlexCleaner/MkvMergeTool.cs b/PlexCleaner/MkvMergeTool.cs index f7b1472d..4223b4e4 100644 --- a/PlexCleaner/MkvMergeTool.cs +++ b/PlexCleaner/MkvMergeTool.cs @@ -262,7 +262,7 @@ string outputName .InputOptions(options => options.Default().SelectTracks(selectMediaProps.Selected).InputFile(inputName) ) - .OutputOptions(options => options.TestSnippets().OutputFile(outputName)) + .OutputOptions(options => options.OutputFile(outputName)) .Build(); // Execute command @@ -279,7 +279,7 @@ public bool ReMuxToMkv(string inputName, string outputName) Command command = GetBuilder() .GlobalOptions(options => options.Default()) .InputOptions(options => options.Default().InputFile(inputName)) - .OutputOptions(options => options.TestSnippets().OutputFile(outputName)) + .OutputOptions(options => options.OutputFile(outputName)) .Build(); // Execute command @@ -296,7 +296,7 @@ public bool RemoveSubtitles(string inputName, string outputName) Command command = GetBuilder() .GlobalOptions(options => options.Default()) .InputOptions(options => options.Default().NoSubtitles().InputFile(inputName)) - .OutputOptions(options => options.TestSnippets().OutputFile(outputName)) + .OutputOptions(options => options.OutputFile(outputName)) .Build(); // Execute command @@ -330,7 +330,7 @@ string outputName .SelectTracks(keepTwo) .InputFile(sourceTwo) ) - .OutputOptions(options => options.TestSnippets().OutputFile(outputName)) + .OutputOptions(options => options.OutputFile(outputName)) .Build(); // Execute command diff --git a/README.md b/README.md index ed4de920..1fa5b36b 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ Utility to optimize media files for Direct Play in Plex, Emby, Jellyfin, etc. **Summary:** -- Treat non-monotonic DTS errors as a verify failure, and attempt to repair it losslessly with the `setts` bitstream filter. +- Treat a non-monotonic DTS as a verify failure, and attempt to repair it losslessly with the `setts` bitstream filter. - Switched closed caption detection to `ffprobe -analyze_frames`, and consolidated the bitrate and DTS packet analyses into a single packet pass. - Added the `DtsTimestampRepair` example plugin that attempts non-monotonic DTS repairs on `RepairFailed` files. From 68331a160b914abe0b61d8b9f57fa63d704f0aff Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 14 Jul 2026 05:45:39 +0000 Subject: [PATCH 07/19] Bump softprops/action-gh-release in the actions-deps group (#838) Bumps the actions-deps group with 1 update: [softprops/action-gh-release](https://github.com/softprops/action-gh-release). Updates `softprops/action-gh-release` from 3.0.1 to 3.0.2 - [Release notes](https://github.com/softprops/action-gh-release/releases) - [Changelog](https://github.com/softprops/action-gh-release/blob/master/CHANGELOG.md) - [Commits](https://github.com/softprops/action-gh-release/compare/718ea10b132b3b2eba29c1007bb80653f286566b...3d0d9888cb7fd7b750713d6e236d1fcb99157228) --- updated-dependencies: - dependency-name: softprops/action-gh-release dependency-version: 3.0.2 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions-deps ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/build-release-task.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-release-task.yml b/.github/workflows/build-release-task.yml index 151d28ae..b4b6edf3 100644 --- a/.github/workflows/build-release-task.yml +++ b/.github/workflows/build-release-task.yml @@ -143,7 +143,7 @@ jobs: # fail_on_unmatched_files catches a missing or misnamed PlexCleaner.7z. - name: Create GitHub release step if: ${{ steps.release-exists.outputs.exists == 'false' || github.event_name == 'workflow_dispatch' }} - uses: softprops/action-gh-release@718ea10b132b3b2eba29c1007bb80653f286566b # v3.0.1 + uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228 # v3.0.2 with: generate_release_notes: true tag_name: ${{ needs.get-version.outputs.SemVer2 }} From cceb70b39b619fb7f33651dc1566617021de1b87 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Tue, 14 Jul 2026 06:54:38 -0700 Subject: [PATCH 08/19] Scope setts to audio DTS and verify A/V sync in the repair gate (#840) * Scope setts to audio DTS and verify A/V sync in the repair gate Two refinements to the lossless timestamp repair: - Attempt setts only when every non-monotonic DTS is on an audio stream. setts is audio-only (a video setts would reorder B-frames), so a video or subtitle DTS is skipped straight to RepairFailed with no wasted rewrite. DtsInfo records each stream's codec type to decide this. This also fixes a latent false positive: DtsInfo fell back to PTS when a packet had no DTS, but Matroska video stores no DTS and its display PTS is legitimately non-monotonic for B-frames; assess only real DTS now. - The gate compared only the payload hash, which proves the samples are unchanged but not their timing. Also compare each stream's start and duration and reject a repair that shifts any stream beyond the A/V-sync tolerance, so a timestamp nudge can never drift audio out of sync. Validated on the corpus: audio-DTS files (Eureka, Ghosted) repair and the sync gate accepts them; video-DTS (Love Island, 50 First Dates) and post-decode (Diplo) skip setts and stay RepairFailed. * Fail the sync gate when a stream timing is present on only one side WithinSyncTolerance passed whenever either side was NaN, so an asymmetric missing start_time or duration slipped through unverified. Pass only when both sides are NaN (symmetric, uncomparable); a value on just one side now fails the gate. --- HISTORY.md | 4 +- PlexCleaner/DtsInfo.cs | 29 ++++++++-- PlexCleaner/FfMpegToolJsonSchema.cs | 24 +++++++++ PlexCleaner/FfProbeTool.cs | 47 ++++++++++++++++ PlexCleaner/ProcessFile.cs | 58 ++++++++++++++++---- PlexCleanerTests/DtsInfoTests.cs | 59 ++++++++++++++++++++- PlexCleanerTests/StreamTimingsProbeTests.cs | 41 ++++++++++++++ 7 files changed, 244 insertions(+), 18 deletions(-) create mode 100644 PlexCleanerTests/StreamTimingsProbeTests.cs diff --git a/HISTORY.md b/HISTORY.md index 826bdb0e..4e74fa0e 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -11,8 +11,8 @@ Utility to optimize media files for Direct Play in Plex, Emby, Jellyfin, etc. - Verify now classifies the decode diagnostics deterministically as clean, a timestamp-only failure, or a decode error; a timestamp-only failure is repaired losslessly when the break is demux-visible and otherwise stays reported, and everything else fails (fail-closed, so an unrecognized diagnostic fails as a decode error). - The classification streams the output line by line, so memory stays bounded even when a file emits a warning per packet ([#827](https://github.com/ptr727/PlexCleaner/issues/827)). - Added a lossless timestamp repair as the first repair tier. - - When verification detects a demux-visible non-monotonic DTS, the audio packet timestamps are rewritten to be strictly monotonic using the `setts` bitstream filter with a stream copy (no re-encode), then re-verified. - - A regression gate compares the per-stream coded payload hash before and after and discards the result unless every stream is byte-identical, so the repair can never alter the media. The full re-encode repair remains for genuine decode corruption. + - When verification detects a demux-visible non-monotonic DTS on an audio stream, the audio packet timestamps are rewritten to be strictly monotonic using the `setts` bitstream filter with a stream copy (no re-encode), then re-verified. A video-stream DTS is not audio-repairable (a video `setts` would reorder B-frames) and stays reported. + - A regression gate compares the per-stream coded payload hash and the per-stream start and duration before and after, discarding the result unless every stream is byte-identical and no stream shifted beyond the A/V-sync tolerance, so the repair can neither alter the media nor drift the audio out of sync. The full re-encode repair remains for genuine decode corruption. - Consolidated the bitrate and DTS packet analyses into a single `ffprobe -show_packets` pass, computing the per-second bitrate and the per-stream DTS monotonicity together instead of reading packets twice. - Switched closed caption detection to `ffprobe -analyze_frames -show_entries stream=closed_captions`, replacing the `movie=...[out0+subcc]` lavfi filter and its QuickScan snippet-remux workaround; QuickScan now bounds the scan with `-read_intervals`. - Added the `DtsTimestampRepair` example plugin. diff --git a/PlexCleaner/DtsInfo.cs b/PlexCleaner/DtsInfo.cs index 705a0e5e..6a88734f 100644 --- a/PlexCleaner/DtsInfo.cs +++ b/PlexCleaner/DtsInfo.cs @@ -8,27 +8,46 @@ public sealed class DtsInfo // Count of non-monotonic packets per stream index private readonly Dictionary _nonMonotonicByStream = []; + // Codec type per stream index, used to decide whether the audio setts filter can repair the DTS + private readonly Dictionary _codecTypeByStream = []; + // Stream indexes carrying a non-monotonic DTS, with the per-stream count public IReadOnlyDictionary NonMonotonicByStream => _nonMonotonicByStream; // True if any stream carries a non-monotonic DTS public bool HasNonMonotonicDts => _nonMonotonicByStream.Count > 0; + // True when every stream carrying a non-monotonic DTS is audio, so the audio-only setts filter can + // repair it; a video or subtitle DTS is not audio-repairable (a video setts would reorder B-frames) + public bool NonMonotonicIsAudioOnly => + HasNonMonotonicDts + && _nonMonotonicByStream.Keys.All(index => + _codecTypeByStream.TryGetValue(index, out string? codecType) + && codecType.Equals("audio", StringComparison.OrdinalIgnoreCase) + ); + public void Add(FfMpegToolJsonSchema.Packet packet) { - // Fall back to PTS when DTS is absent, matching how the muxer derives DTS - double dts = !double.IsNaN(packet.DtsTime) ? packet.DtsTime : packet.PtsTime; - if (double.IsNaN(dts)) + // Assess only packets that carry a real DTS. A missing DTS (e.g. Matroska video, which stores no + // DTS) is reconstructed by the muxer from PTS, whose display order is legitimately non-monotonic + // for B-frames, so it cannot be judged from packet data and must not be flagged + if (double.IsNaN(packet.DtsTime)) { return; } + // Record the codec type so a flagged stream can be classified as audio or not + _codecTypeByStream[packet.StreamIndex] = packet.CodecType; + // Flag a non-increasing DTS relative to the previous packet in the same stream - if (_lastDts.TryGetValue(packet.StreamIndex, out double previous) && dts <= previous) + if ( + _lastDts.TryGetValue(packet.StreamIndex, out double previous) + && packet.DtsTime <= previous + ) { _nonMonotonicByStream[packet.StreamIndex] = _nonMonotonicByStream.GetValueOrDefault(packet.StreamIndex) + 1; } - _lastDts[packet.StreamIndex] = dts; + _lastDts[packet.StreamIndex] = packet.DtsTime; } } diff --git a/PlexCleaner/FfMpegToolJsonSchema.cs b/PlexCleaner/FfMpegToolJsonSchema.cs index 473685fb..ad8ec5ed 100644 --- a/PlexCleaner/FfMpegToolJsonSchema.cs +++ b/PlexCleaner/FfMpegToolJsonSchema.cs @@ -41,6 +41,29 @@ public class ClosedCaptionsTrack public int ClosedCaptions { get; set; } } + // Per-stream start and duration, used to verify a timestamp repair did not shift A/V sync + public class StreamTimingsProbe + { + [JsonPropertyName("streams")] + public List Streams { get; } = []; + + public static StreamTimingsProbe FromJson(string json) => + JsonSerializer.Deserialize(json, FfMpegToolJsonContext.Default.StreamTimingsProbe) + ?? throw new JsonException("Failed to deserialize StreamTimingsProbe"); + } + + public class StreamTiming + { + [JsonPropertyName("index")] + public int Index { get; set; } + + [JsonPropertyName("start_time")] + public double StartTime { get; set; } = double.NaN; + + [JsonPropertyName("duration")] + public double Duration { get; set; } = double.NaN; + } + public class FormatInfo { [JsonPropertyName("format_name")] @@ -172,4 +195,5 @@ public class Packet [JsonSerializable(typeof(FfMpegToolJsonSchema.FfProbe))] [JsonSerializable(typeof(FfMpegToolJsonSchema.Packet))] [JsonSerializable(typeof(FfMpegToolJsonSchema.ClosedCaptionsProbe))] +[JsonSerializable(typeof(FfMpegToolJsonSchema.StreamTimingsProbe))] internal partial class FfMpegToolJsonContext : JsonSerializerContext; diff --git a/PlexCleaner/FfProbeTool.cs b/PlexCleaner/FfProbeTool.cs index e89b1414..51c39599 100644 --- a/PlexCleaner/FfProbeTool.cs +++ b/PlexCleaner/FfProbeTool.cs @@ -237,6 +237,53 @@ public bool GetClosedCaptions(string fileName, out bool hasClosedCaptions) return true; } + public bool GetStreamTimings( + string fileName, + out Dictionary timings + ) + { + // Per-stream start and duration, used to verify a timestamp repair preserved A/V sync + timings = []; + Command command = GetBuilder() + .GlobalOptions(options => options.Default().HideBanner().LogLevelError()) + .FfProbeOptions(options => + options + .ShowStreams() + .ShowEntries("stream=index,start_time,duration") + .OutputFormatJson() + .InputFile(fileName) + ) + .Build(); + + // Execute command + Log.Debug("Getting stream timings : {FileName}", fileName); + if (!Execute(command, false, true, out BufferedCommandResult result)) + { + return false; + } + if (result.ExitCode != 0) + { + Log.Error("Failed to get stream timings : {FileName}", fileName); + return LogFailedResult(result); + } + + // FromJson throws on malformed output + try + { + FfMpegToolJsonSchema.StreamTimingsProbe probe = + FfMpegToolJsonSchema.StreamTimingsProbe.FromJson(result.StandardOutput); + foreach (FfMpegToolJsonSchema.StreamTiming stream in probe.Streams) + { + timings[stream.Index] = (stream.StartTime, stream.Duration); + } + } + catch (Exception e) when (Log.Logger.LogAndHandle(e)) + { + return false; + } + return true; + } + public bool GetAnalysisPackets( string fileName, Func packetFunc, diff --git a/PlexCleaner/ProcessFile.cs b/PlexCleaner/ProcessFile.cs index 53326ebf..fc17a5d4 100644 --- a/PlexCleaner/ProcessFile.cs +++ b/PlexCleaner/ProcessFile.cs @@ -2214,12 +2214,13 @@ private bool RepairTimestampsAndSetState(ref bool modified) private bool TryLosslessTimestampRepair() { - // Only a demux-visible non-monotonic DTS can be rewritten losslessly, a post-decode-only break has - // no demux target and stays a reported failure, an analysis failure is treated as unrepaired too + // The audio-only setts filter can repair only an audio-stream DTS, so attempt it only when every + // non-monotonic stream is audio; a video or subtitle DTS, a post-decode-only break with no demux + // target, or an analysis failure all stay reported failures without a wasted rewrite if ( !GetPacketAnalysis(false, out _, out DtsInfo? dtsInfo) || dtsInfo == null - || !dtsInfo.HasNonMonotonicDts + || !dtsInfo.NonMonotonicIsAudioOnly ) { return false; @@ -2253,13 +2254,52 @@ private bool TryLosslessTimestampRepair() return true; } - private static bool TimestampRepairRegressionGate(string original, string repaired) => - // Lossless requires every stream's coded payload to be byte-identical before and after - // The streamhash muxer hashes packet data only, so a matching hash proves only timestamps changed - Tools.FfMpeg.GetStreamHashes(original, out Dictionary before) - && Tools.FfMpeg.GetStreamHashes(repaired, out Dictionary after) + // A timestamp nudge must not shift a stream's start or duration by more than the A/V-sync + // perceptibility threshold, so the repair never introduces audible drift + private const double SyncToleranceSeconds = 0.040; + + private static bool TimestampRepairRegressionGate(string original, string repaired) + { + // Payload must be byte-identical; the streamhash muxer hashes packet data only, not timestamps, + // so a matching hash proves only the timestamps changed + if ( + !Tools.FfMpeg.GetStreamHashes(original, out Dictionary beforeHash) + || !Tools.FfMpeg.GetStreamHashes(repaired, out Dictionary afterHash) + || beforeHash.Count != afterHash.Count + || !beforeHash.All(kvp => + afterHash.TryGetValue(kvp.Key, out string? hash) && hash == kvp.Value + ) + ) + { + return false; + } + + // Timing must be preserved; the streamhash proves the samples are identical but not where they + // play, so verify no stream's start or duration moved beyond the A/V-sync tolerance + return TimestampRepairSyncPreserved(original, repaired); + } + + private static bool TimestampRepairSyncPreserved(string original, string repaired) => + Tools.FfProbe.GetStreamTimings( + original, + out Dictionary before + ) + && Tools.FfProbe.GetStreamTimings( + repaired, + out Dictionary after + ) && before.Count == after.Count - && before.All(kvp => after.TryGetValue(kvp.Key, out string? hash) && hash == kvp.Value); + && before.All(kvp => + after.TryGetValue(kvp.Key, out (double Start, double Duration) other) + && WithinSyncTolerance(kvp.Value.Start, other.Start) + && WithinSyncTolerance(kvp.Value.Duration, other.Duration) + ); + + private static bool WithinSyncTolerance(double before, double after) => + // A value present on only one side cannot be verified, so fail closed; both sides missing (NaN) + // is symmetric and uncomparable, treat as unchanged; otherwise the shift must be within tolerance + double.IsNaN(before) == double.IsNaN(after) + && (double.IsNaN(before) || Math.Abs(after - before) <= SyncToleranceSeconds); public bool SetLastWriteTimeUtc(DateTime lastWriteTimeUtc) { diff --git a/PlexCleanerTests/DtsInfoTests.cs b/PlexCleanerTests/DtsInfoTests.cs index 650aaae5..0b9b10f5 100644 --- a/PlexCleanerTests/DtsInfoTests.cs +++ b/PlexCleanerTests/DtsInfoTests.cs @@ -6,8 +6,17 @@ namespace PlexCleanerTests; public class DtsInfoTests { - private static FfMpegToolJsonSchema.Packet Packet(long streamIndex, double dtsTime) => - new() { StreamIndex = streamIndex, DtsTime = dtsTime }; + private static FfMpegToolJsonSchema.Packet Packet( + long streamIndex, + double dtsTime, + string codecType = "audio" + ) => + new() + { + StreamIndex = streamIndex, + DtsTime = dtsTime, + CodecType = codecType, + }; [Fact] public void Add_MonotonicDts_NoDetection() @@ -58,4 +67,50 @@ public void Add_NanDts_Ignored() _ = dtsInfo.HasNonMonotonicDts.Should().BeFalse(); } + + [Fact] + public void NonMonotonicIsAudioOnly_AudioDts_True() + { + // A demux-visible audio DTS is repairable by the audio setts filter + DtsInfo dtsInfo = new(); + dtsInfo.Add(Packet(1, 0.08, "audio")); + dtsInfo.Add(Packet(1, 0.08, "audio")); + + _ = dtsInfo.NonMonotonicIsAudioOnly.Should().BeTrue(); + } + + [Fact] + public void NonMonotonicIsAudioOnly_VideoDts_False() + { + // A video DTS is not audio-repairable, a video setts would reorder B-frames + DtsInfo dtsInfo = new(); + dtsInfo.Add(Packet(0, 0.08, "video")); + dtsInfo.Add(Packet(0, 0.08, "video")); + + _ = dtsInfo.HasNonMonotonicDts.Should().BeTrue(); + _ = dtsInfo.NonMonotonicIsAudioOnly.Should().BeFalse(); + } + + [Fact] + public void NonMonotonicIsAudioOnly_MixedAudioAndVideo_False() + { + // If any offending stream is non-audio the audio setts cannot fully repair the file + DtsInfo dtsInfo = new(); + dtsInfo.Add(Packet(0, 0.08, "video")); + dtsInfo.Add(Packet(0, 0.08, "video")); + dtsInfo.Add(Packet(1, 0.08, "audio")); + dtsInfo.Add(Packet(1, 0.08, "audio")); + + _ = dtsInfo.NonMonotonicIsAudioOnly.Should().BeFalse(); + } + + [Fact] + public void NonMonotonicIsAudioOnly_NoDts_False() + { + DtsInfo dtsInfo = new(); + dtsInfo.Add(Packet(1, 0.04, "audio")); + dtsInfo.Add(Packet(1, 0.08, "audio")); + + _ = dtsInfo.NonMonotonicIsAudioOnly.Should().BeFalse(); + } } diff --git a/PlexCleanerTests/StreamTimingsProbeTests.cs b/PlexCleanerTests/StreamTimingsProbeTests.cs new file mode 100644 index 00000000..9de77105 --- /dev/null +++ b/PlexCleanerTests/StreamTimingsProbeTests.cs @@ -0,0 +1,41 @@ +using AwesomeAssertions; +using PlexCleaner; +using Xunit; + +namespace PlexCleanerTests; + +public class StreamTimingsProbeTests +{ + [Fact] + public void FromJson_StartAndDuration_Parsed() + { + // ffprobe emits start_time and duration as strings, parsed to double + // lang=json + const string json = """ + { "streams": [ { "index": 0, "start_time": "0.000000", "duration": "1234.567000" } ] } + """; + FfMpegToolJsonSchema.StreamTimingsProbe probe = + FfMpegToolJsonSchema.StreamTimingsProbe.FromJson(json); + + _ = probe.Streams.Should().ContainSingle(); + _ = probe.Streams[0].Index.Should().Be(0); + _ = probe.Streams[0].StartTime.Should().Be(0.0); + _ = probe.Streams[0].Duration.Should().Be(1234.567); + } + + [Fact] + public void FromJson_MissingTiming_DefaultsToNaN() + { + // A stream without start_time or duration keeps the NaN sentinel; the gate treats it as unchanged + // only when both sides are NaN, and fails closed when a value is present on only one side + // lang=json + const string json = """ + { "streams": [ { "index": 1 } ] } + """; + FfMpegToolJsonSchema.StreamTimingsProbe probe = + FfMpegToolJsonSchema.StreamTimingsProbe.FromJson(json); + + _ = double.IsNaN(probe.Streams[0].StartTime).Should().BeTrue(); + _ = double.IsNaN(probe.Streams[0].Duration).Should().BeTrue(); + } +} From 71465d8e74ca1bf9e004ea678345f99d7c445991 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Tue, 14 Jul 2026 11:01:54 -0700 Subject: [PATCH 09/19] AGENTS: add author-identity git rule (#841) Add the fleet author-identity rule to the git-governance section, immediately after the signing bullet. Commits must use the committing account's own GitHub noreply identity (ptr727@users.noreply.github.com for this fleet), never a private, personal, or invented address. A wrong identity trips GitHub's email-privacy push protection (GH007) or pollutes history with an unrecognized author. This brings AGENTS.md into line with the canonical template, which already carried the signing and default-to-staging rules alongside this one. Co-authored-by: Claude Opus 4.8 (1M context) --- AGENTS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/AGENTS.md b/AGENTS.md index 58711111..2c9aa55a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -10,6 +10,7 @@ This file is the canonical reference for cross-cutting AI-agent rules. The CI/CD - **Default to staging, not committing.** Stage changes with `git add` and leave `git commit` to the developer unless the developer has explicitly authorized the agent to commit for the current ask ("commit this", "open a PR", etc.). Authorization is scope-bound - it covers the commits needed for that specific task, not a blanket commit license for the rest of the session. - **All commits must be cryptographically signed (SSH or GPG).** Branch protection enforces this on both branches; unsigned commits are rejected on push. Signing depends on environment configuration - `git config commit.gpgsign true`, a configured `user.signingkey`, and a working signing agent (loaded `ssh-agent` for SSH, or `gpg-agent` for GPG). If signing is not configured in the environment, **do not commit** - surface the missing config to the developer and stop at `git add`. Verify before any agent-authored commit (`git config --get commit.gpgsign && ssh-add -L` or the GPG equivalent). **Signing must be live before the *first* commit, not retrofitted.** Turning on `Require signed commits` against a branch that already has unsigned commits forces a rewrite of that entire history to re-sign it - changing every commit SHA and making whoever does the rewrite the committer and signer of every commit (a rebase preserves the `author` field but not the original signatures; you cannot sign another contributor's commits for them). During new-repo setup, never create commits until signing is verified. +- **Commit under the committing account's own GitHub `noreply` identity - never a private, personal, or invented address.** The `author` and `committer` on every agent-authored commit are the GitHub `noreply` address of the account whose key signs the commit (above) - GitHub issues these in a `username@users.noreply.github.com` or `ID+username@users.noreply.github.com` form, and for this single-maintainer fleet it is the owner's `ptr727@users.noreply.github.com`. Do not set `user.name`/`user.email` to a fabricated persona, bot name, or product name, and do not commit under whatever identity the environment happens to carry: verify `git config --get user.email` is that GitHub `noreply` address before committing, and fix it if not. A wrong identity is not cosmetic - a private email trips GitHub's email-privacy push protection (GH007), and an unrecognized or invented author pollutes history. Identity is separate from signing: a wrong author does not by itself fail the signature rule, but the ad-hoc identities that produce it are typically also unsigned, which the signing rule above then rejects on push. - **Never force push.** Do not run `git push --force` or `git push --force-with-lease` under any circumstances. Force pushing rewrites shared history and can cause data loss. - **Never run destructive git commands** (`git reset --hard`, `git checkout .`, `git restore .`, `git clean -f`) without explicit developer instruction. From 054d273f594729ff0daa91bf797a337b351de8b0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 15 Jul 2026 05:47:46 +0000 Subject: [PATCH 10/19] Bump the nuget-deps group with 5 updates (#843) Bumps Microsoft.NET.Test.Sdk from 18.7.0 to 18.8.1 Bumps Microsoft.SourceLink.GitHub from 10.0.300 to 10.0.301 Bumps ptr727.LanguageTags from 1.5.39 to 1.5.44 Bumps ptr727.Utilities from 4.0.7 to 4.0.15 Bumps System.CommandLine from 2.0.9 to 2.0.10 --- updated-dependencies: - dependency-name: Microsoft.NET.Test.Sdk dependency-version: 18.8.1 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: nuget-deps - dependency-name: Microsoft.SourceLink.GitHub dependency-version: 10.0.301 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: nuget-deps - dependency-name: ptr727.LanguageTags dependency-version: 1.5.44 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: nuget-deps - dependency-name: ptr727.Utilities dependency-version: 4.0.15 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: nuget-deps - dependency-name: System.CommandLine dependency-version: 2.0.10 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: nuget-deps ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Directory.Packages.props | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 63774c59..4386353c 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -3,18 +3,18 @@ - - + + - - + + - + From aa49adcd9f66b6c5f1f3ee285d225e573a2c6761 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Wed, 15 Jul 2026 09:00:31 -0700 Subject: [PATCH 11/19] Fix DTS repair escalation and idet parse regressions (#844) * Parse idet stat lines independently to tolerate interleaved output The interlace detector required the three idet stat lines (Repeated Fields / Single frame / Multi frame) to be a single contiguous block. ffmpeg interleaves other stderr lines between them: on a source with non-monotonic DTS the -f null muxer emits "non monotonically increasing dts to muxer" warnings (idet uses -fflags +genpts), and a full-file scan decodes enough packets that a warning lands between the stat lines. The contiguous match then finds nothing, so the parse fails and the whole file is aborted, skipping the remux/re-encode that would repair it. Match each stat line independently and take the last of each: robust to interleaved warnings, and it still selects the final cumulative counts over idet's early all-zero pass. Log the raw idet output on a parse failure so any remaining unexpected case is diagnosable rather than guessed at. An idet failure stays a hard error that aborts the file: it is unexpected, so it should surface as a bug to fix, not be masked. Adds a regression test with warnings interspersed between the stat lines. Co-Authored-By: Claude Opus 4.8 (1M context) * Escalate a DTS repair through remux and re-encode before failing A non-monotonic DTS verify failure only tried the lossless audio setts repair; when that did not apply (a video-stream or post-decode DTS) the file was marked RepairFailed with no further attempt. A full re-encode rebuilds the timestamps and does fix these, which the pre-3.21 pipeline relied on, so several files that used to repair now fail. Make the DTS path escalate through the standard repair tiers: surgical lossless setts, then a plain remux, then a re-encode, marking Repaired on the first tier whose re-verify is clean and RepairFailed only when all tiers fail. Add the missing remux tier to RepairAndReVerify so both the DTS and decode-error paths share the same surgical -> remux -> re-encode escalation. Co-Authored-By: Claude Opus 4.8 (1M context) * Log per-file processing time in the driver Time each file's task in ProcessDriver and report it on the "After" line, with the elapsed before the file name (file name last, being longest). Share one HH:mm:ss.fff formatter with the run summary so both read the same; hours come from TotalHours so a multi-day run does not wrap. Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Claude Opus 4.8 (1M context) --- PlexCleaner/FfMpegIdetInfo.cs | 65 ++++++++++------- PlexCleaner/ProcessDriver.cs | 14 +++- PlexCleaner/ProcessFile.cs | 58 +++++++++++++-- PlexCleanerTests/FfMpegIdetParsingTests.cs | 82 ++++++++++------------ PlexCleanerTests/ProcessDriverTests.cs | 35 +++++++++ 5 files changed, 173 insertions(+), 81 deletions(-) create mode 100644 PlexCleanerTests/ProcessDriverTests.cs diff --git a/PlexCleaner/FfMpegIdetInfo.cs b/PlexCleaner/FfMpegIdetInfo.cs index ebb5d1be..822c697f 100644 --- a/PlexCleaner/FfMpegIdetInfo.cs +++ b/PlexCleaner/FfMpegIdetInfo.cs @@ -105,43 +105,56 @@ internal bool Parse(string text) // [out#0/null @ 000001c11d401040] video:32843KiB audio:0KiB subtitle:0KiB other streams:0KiB global headers:0KiB muxing overhead: unknown // frame=76434 fps=1114 q=-0.0 Lsize=N/A time=00:42:30.68 bitrate=N/A speed=37.2x - // Match (regex construction uses \n for new line) - // idet can emit its stats more than once (an early empty pass before the final counts), - // so match every triple and use the last, which holds the final cumulative counts - MatchCollection matches = IdetRegex() - .Matches(text.Replace("\r\n", "\n", StringComparison.Ordinal)); - if (matches.Count == 0) + // Match each of the three stat lines independently and take the last of each. idet emits its + // stats more than once (an early empty pass before the final cumulative counts), and ffmpeg can + // interleave other stderr lines between them (e.g. -f null muxer "non monotonically increasing + // dts" warnings when the source has non-monotonic DTS), so a single contiguous three-line match + // is unreliable; the last of each line is the final cumulative value + string normalized = text.Replace("\r\n", "\n", StringComparison.Ordinal); + Match? repeated = LastMatch(RepeatedFieldsRegex(), normalized); + Match? single = LastMatch(SingleFrameRegex(), normalized); + Match? multi = LastMatch(MultiFrameRegex(), normalized); + if (repeated == null || single == null || multi == null) { - Log.Error("Failed to parse idet output"); + // Log the output that failed to parse so the failure is diagnosable, joined to one line + Log.Error("Failed to parse idet output : {Output}", text.ReplaceLineEndings(" | ")); return false; } - Match match = matches[^1]; // Get the frame counts - RepeatedFields.Neither = ParseGroupInt(match, "repeated_neither"); - RepeatedFields.Top = ParseGroupInt(match, "repeated_top"); - RepeatedFields.Bottom = ParseGroupInt(match, "repeated_bottom"); - - SingleFrame.Tff = ParseGroupInt(match, "single_tff"); - SingleFrame.Bff = ParseGroupInt(match, "single_bff"); - SingleFrame.Progressive = ParseGroupInt(match, "single_prog"); - SingleFrame.Undetermined = ParseGroupInt(match, "single_und"); - - MultiFrame.Tff = ParseGroupInt(match, "multi_tff"); - MultiFrame.Bff = ParseGroupInt(match, "multi_bff"); - MultiFrame.Progressive = ParseGroupInt(match, "multi_prog"); - MultiFrame.Undetermined = ParseGroupInt(match, "multi_und"); + RepeatedFields.Neither = ParseGroupInt(repeated, "repeated_neither"); + RepeatedFields.Top = ParseGroupInt(repeated, "repeated_top"); + RepeatedFields.Bottom = ParseGroupInt(repeated, "repeated_bottom"); + + SingleFrame.Tff = ParseGroupInt(single, "single_tff"); + SingleFrame.Bff = ParseGroupInt(single, "single_bff"); + SingleFrame.Progressive = ParseGroupInt(single, "single_prog"); + SingleFrame.Undetermined = ParseGroupInt(single, "single_und"); + + MultiFrame.Tff = ParseGroupInt(multi, "multi_tff"); + MultiFrame.Bff = ParseGroupInt(multi, "multi_bff"); + MultiFrame.Progressive = ParseGroupInt(multi, "multi_prog"); + MultiFrame.Undetermined = ParseGroupInt(multi, "multi_und"); return true; } + private static Match? LastMatch(Regex regex, string text) + { + MatchCollection matches = regex.Matches(text); + return matches.Count > 0 ? matches[^1] : null; + } + internal static int ParseGroupInt(Match match, string groupName) => int.Parse(match.Groups[groupName].Value.Trim(), CultureInfo.InvariantCulture); - [GeneratedRegex( - $"{IdetRepeatedFields}\n{IdetSingleFrame}\n{IdetMultiFrame}", - RegexOptions.IgnoreCase | RegexOptions.Multiline - )] - public static partial Regex IdetRegex(); + [GeneratedRegex(IdetRepeatedFields, RegexOptions.IgnoreCase | RegexOptions.Multiline)] + private static partial Regex RepeatedFieldsRegex(); + + [GeneratedRegex(IdetSingleFrame, RegexOptions.IgnoreCase | RegexOptions.Multiline)] + private static partial Regex SingleFrameRegex(); + + [GeneratedRegex(IdetMultiFrame, RegexOptions.IgnoreCase | RegexOptions.Multiline)] + private static partial Regex MultiFrameRegex(); public class Repeated { diff --git a/PlexCleaner/ProcessDriver.cs b/PlexCleaner/ProcessDriver.cs index a959d975..b4545247 100644 --- a/PlexCleaner/ProcessDriver.cs +++ b/PlexCleaner/ProcessDriver.cs @@ -178,8 +178,10 @@ Func taskFunc fileName ); - // Perform the task + // Perform the task, timing this file's work + long startTimestamp = Stopwatch.GetTimestamp(); bool taskResult = taskFunc(fileName); + TimeSpan taskElapsed = Stopwatch.GetElapsedTime(startTimestamp); // Handle cancel request Program.CancelToken().ThrowIfCancellationRequested(); @@ -198,9 +200,10 @@ Func taskFunc totalCount ); Log.Information( - "{TaskName} ({Processed:F2}%) After : {FileName}", + "{TaskName} ({Processed:F2}%) Elapsed : {Elapsed:l} : After : {FileName}", taskName, processedPercentage, + FormatDuration(taskElapsed), fileName ); } @@ -222,7 +225,8 @@ Func taskFunc // Done, force logging so the summary survives the warning floor and an interrupted run Log.Logger.LogOverrideContext().Information("Completed {TaskName}", taskName); - Log.Logger.LogOverrideContext().Information("Processing time : {Elapsed}", timer.Elapsed); + Log.Logger.LogOverrideContext() + .Information("Processing time : {Elapsed:l}", FormatDuration(timer.Elapsed)); Log.Logger.LogOverrideContext().Information("Total files : {Count}", totalCount); Log.Logger.LogOverrideContext().Information("Error files : {Count}", errorCount); @@ -477,6 +481,10 @@ public static bool GetToolInfo(List fileList) => } ); + internal static string FormatDuration(TimeSpan value) => + // Consistent HH:mm:ss.fff across log entries; hours from TotalHours so a multi-day run never wraps + $"{(int)value.TotalHours:D2}:{value.Minutes:D2}:{value.Seconds:D2}.{value.Milliseconds:D3}"; + private static double GetPercentage(int dividend, int divisor) { // Calculate double digit precision avoiding 100% until really complete diff --git a/PlexCleaner/ProcessFile.cs b/PlexCleaner/ProcessFile.cs index fc17a5d4..e2926d63 100644 --- a/PlexCleaner/ProcessFile.cs +++ b/PlexCleaner/ProcessFile.cs @@ -1186,7 +1186,7 @@ out FfMpegIdetInfo? idetInfo // Count the frame types using the idet filter if (!GetIdetInfo(out idetInfo) || idetInfo == null) { - // Error + // Error, an idet execution or parse failure is unexpected, abort the file so the bug surfaces return false; } @@ -1858,8 +1858,8 @@ public bool VerifyAndRepair(ref bool modified) Debug.Assert(!_sidecarFile.State.HasFlag(SidecarFile.StatesType.Verified)); Debug.Assert(!_sidecarFile.State.HasFlag(SidecarFile.StatesType.Repaired)); - // Non-monotonic DTS is a repairable failure, fix it losslessly with setts - // A re-encode cannot fix timestamps so it is not a fallback here + // Non-monotonic DTS is a repairable failure; try a lossless surgical setts repair first, then fall + // through to the shared remux and re-encode tiers for a video or post-decode DTS setts cannot fix if (_lastVerifyResult == VerifyResult.TimestampOnly) { return RepairTimestampsAndSetState(ref modified); @@ -2074,7 +2074,20 @@ private bool RepairAndReVerify() // https://ffmpeg.org/ffmpeg-filters.html#crop // -vf crop='iw-mod(iw,4)':'ih-mod(ih,4)' - // Repair to temp file, only if verify is successful replace original file + // Tier 1: a plain remux rewrites the container and its timestamps, clearing a demux-visible break + // such as a non-monotonic DTS without re-encoding; it cannot fix decode-level corruption + if (TryRemuxRepair()) + { + Log.Information("Repair succeeded : {FileName}", FileInfo.FullName); + return true; + } + if (Program.IsCancelledError()) + { + return false; + } + + // Tier 2: re-encode rebuilds the streams, fixing decode corruption and timestamp breaks a remux + // cannot. Repair to temp file, only if verify is successful replace original file string tempName = Path.ChangeExtension(FileInfo.FullName, ".tmp12"); Debug.Assert(FileInfo.FullName != tempName); @@ -2148,6 +2161,34 @@ private bool RepairAndReVerify() return true; } + private bool TryRemuxRepair() + { + // Remux to a temp file, only replace the original if the re-verify is clean + string tempName = Path.ChangeExtension(FileInfo.FullName, ".tmp11"); + Debug.Assert(FileInfo.FullName != tempName); + + Log.Information( + "Attempting media repair by remuxing using MkvMerge : {FileName}", + FileInfo.FullName + ); + if (!Tools.MkvMerge.ReMuxToMkv(FileInfo.FullName, tempName)) + { + File.Delete(tempName); + return false; + } + + // Require a clean re-verify, a remux that still fails falls through to the re-encode tier + if (VerifyMediaStreams(new FileInfo(tempName)) != VerifyResult.Clean) + { + File.Delete(tempName); + return false; + } + + // Verify succeeded, replace the original with the remuxed file + File.Move(tempName, FileInfo.FullName, true); + return true; + } + public bool RepairTimestamps(ref bool modified) { // Only process Matroska files, the audio timestamp repair does not require a video stream @@ -2186,8 +2227,10 @@ public bool RepairTimestamps(ref bool modified) private bool RepairTimestampsAndSetState(ref bool modified) { - // A detected non-monotonic DTS is a failure, repair it losslessly when the break is demux-visible - if (TryLosslessTimestampRepair()) + // Escalate through the repair tiers: a lossless surgical setts repair for a demux-visible audio + // DTS, then the shared remux and re-encode ladder for a video or post-decode DTS setts cannot fix. + // The first tier whose re-verify is clean wins + if (TryLosslessTimestampRepair() || RepairAndReVerify()) { _sidecarFile.State |= SidecarFile.StatesType.Verified; _sidecarFile.State &= ~SidecarFile.StatesType.VerifyFailed; @@ -2203,7 +2246,8 @@ private bool RepairTimestampsAndSetState(ref bool modified) return false; } - // A detected DTS we could not repair stays reported as a failure, a detected issue is not cleared + // No tier could repair the detected DTS, it stays reported as a failure, a detected issue is + // not cleared _sidecarFile.State |= SidecarFile.StatesType.VerifyFailed; _sidecarFile.State &= ~SidecarFile.StatesType.Verified; _sidecarFile.State |= SidecarFile.StatesType.RepairFailed; diff --git a/PlexCleanerTests/FfMpegIdetParsingTests.cs b/PlexCleanerTests/FfMpegIdetParsingTests.cs index 4f1e27a8..df3eee2e 100644 --- a/PlexCleanerTests/FfMpegIdetParsingTests.cs +++ b/PlexCleanerTests/FfMpegIdetParsingTests.cs @@ -1,5 +1,4 @@ using System.Diagnostics.CodeAnalysis; -using System.Text.RegularExpressions; using AwesomeAssertions; using PlexCleaner; using PlexCleanerTests; @@ -86,52 +85,45 @@ public class FfMpegIdetParsingTests }, } }, + { + // ffmpeg interleaves other stderr lines between the idet stats; on a source with + // non-monotonic DTS the -f null muxer emits "non monotonically increasing dts" warnings + // between the three lines (the VC1 regression). The three lines must parse independently + new string( + """ + [Parsed_idet_0 @ 0x7ec7fc004280] Repeated Fields: Neither:139809 Top: 2 Bottom: 0 + [null @ 0x5a4c] Application provided invalid, non monotonically increasing dts to muxer in stream 0: 42 >= 42 + [Parsed_idet_0 @ 0x7ec7fc004280] Single frame detection: TFF: 333 BFF: 259 Progressive:138387 Undetermined: 832 + [null @ 0x5a4c] Application provided invalid, non monotonically increasing dts to muxer in stream 0: 43 >= 43 + [Parsed_idet_0 @ 0x7ec7fc004280] Multi frame detection: TFF: 4 BFF: 44 Progressive:139709 Undetermined: 54 + """ + ), + new FfMpegIdetInfo + { + RepeatedFields = new FfMpegIdetInfo.Repeated + { + Neither = 139809, + Top = 2, + Bottom = 0, + }, + SingleFrame = new FfMpegIdetInfo.Frames + { + Tff = 333, + Bff = 259, + Progressive = 138387, + Undetermined = 832, + }, + MultiFrame = new FfMpegIdetInfo.Frames + { + Tff = 4, + Bff = 44, + Progressive = 139709, + Undetermined = 54, + }, + } + }, }; - [Theory] - [MemberData(nameof(Data))] - [SuppressMessage( - "Usage", - "xUnit1045:Avoid using TheoryData type arguments that might not be serializable", - Justification = "FfMpegIdetInfoSerializer" - )] - public void Parse_Idet_Field_Test(string text, FfMpegIdetInfo idetInfo) - { - // Follow same pattern as in FfMpegIdetInfo.Parse() : use the last triple - text = text.Replace("\r\n", "\n", StringComparison.Ordinal); - MatchCollection matches = FfMpegIdetInfo.IdetRegex().Matches(text); - _ = matches.Count.Should().BeGreaterThan(0); - Match match = matches[^1]; - - _ = idetInfo - .RepeatedFields.Neither.Should() - .Be(FfMpegIdetInfo.ParseGroupInt(match, "repeated_neither")); - _ = idetInfo - .RepeatedFields.Top.Should() - .Be(FfMpegIdetInfo.ParseGroupInt(match, "repeated_top")); - _ = idetInfo - .RepeatedFields.Bottom.Should() - .Be(FfMpegIdetInfo.ParseGroupInt(match, "repeated_bottom")); - - _ = idetInfo.SingleFrame.Tff.Should().Be(FfMpegIdetInfo.ParseGroupInt(match, "single_tff")); - _ = idetInfo.SingleFrame.Bff.Should().Be(FfMpegIdetInfo.ParseGroupInt(match, "single_bff")); - _ = idetInfo - .SingleFrame.Progressive.Should() - .Be(FfMpegIdetInfo.ParseGroupInt(match, "single_prog")); - _ = idetInfo - .SingleFrame.Undetermined.Should() - .Be(FfMpegIdetInfo.ParseGroupInt(match, "single_und")); - - _ = idetInfo.MultiFrame.Tff.Should().Be(FfMpegIdetInfo.ParseGroupInt(match, "multi_tff")); - _ = idetInfo.MultiFrame.Bff.Should().Be(FfMpegIdetInfo.ParseGroupInt(match, "multi_bff")); - _ = idetInfo - .MultiFrame.Progressive.Should() - .Be(FfMpegIdetInfo.ParseGroupInt(match, "multi_prog")); - _ = idetInfo - .MultiFrame.Undetermined.Should() - .Be(FfMpegIdetInfo.ParseGroupInt(match, "multi_und")); - } - [Theory] [MemberData(nameof(Data))] [SuppressMessage( diff --git a/PlexCleanerTests/ProcessDriverTests.cs b/PlexCleanerTests/ProcessDriverTests.cs new file mode 100644 index 00000000..a37de092 --- /dev/null +++ b/PlexCleanerTests/ProcessDriverTests.cs @@ -0,0 +1,35 @@ +using AwesomeAssertions; +using PlexCleaner; +using Xunit; + +namespace PlexCleanerTests; + +public class ProcessDriverTests +{ + [Theory] + [InlineData(0, 0, 0, 0, "00:00:00.000")] + [InlineData(0, 0, 12, 345, "00:00:12.345")] + [InlineData(0, 5, 9, 7, "00:05:09.007")] + [InlineData(2, 3, 4, 500, "02:03:04.500")] + public void FormatDuration_MillisecondPrecision( + int hours, + int minutes, + int seconds, + int milliseconds, + string expected + ) + { + TimeSpan value = new(0, hours, minutes, seconds, milliseconds); + + _ = ProcessDriver.FormatDuration(value).Should().Be(expected); + } + + [Fact] + public void FormatDuration_MultiDay_HoursDoNotWrap() + { + // Hours come from TotalHours, so a run past 24h reads as accumulated hours, not a day rollover + TimeSpan value = new(1, 3, 14, 3, 512); + + _ = ProcessDriver.FormatDuration(value).Should().Be("27:14:03.512"); + } +} From b01e78738b23dad8388a0f9a77818f3b54e5b885 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Wed, 15 Jul 2026 10:55:25 -0700 Subject: [PATCH 12/19] Document the README + HISTORY cspell CI scope in CODESTYLE (#846) * Document the README + HISTORY cspell CI scope in CODESTYLE Propagate the CODESTYLE "Spelling CI scope" rule from the template (ptr727/ProjectTemplate#302, #303): the CI spell-check gate covers README.md + HISTORY.md, not all markdown, with broad live checking left to the editor extension. Co-Authored-By: Claude Opus 4.8 (1M context) * Reword the cspell-scope item to not name template-only surfaces Copilot review: the item referenced a Lint: Spelling VS Code task and an AGENTS.md cspell one-liner that exist in the template but not in this repo. Reword generically so the guidance is accurate regardless of which local cspell surfaces a repo actually has. Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Claude Opus 4.8 (1M context) --- CODESTYLE.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CODESTYLE.md b/CODESTYLE.md index 2f1a4735..2c443293 100644 --- a/CODESTYLE.md +++ b/CODESTYLE.md @@ -35,6 +35,7 @@ These apply repo-wide, in every directory: 1. **Markdown linting**: All `.md` files must be lint-clean (error and warning free) via the VS Code `markdownlint` extension. [`.markdownlint-cli2.jsonc`](./.markdownlint-cli2.jsonc) at the repo root is the single source of truth - the davidanson `markdownlint` extension and a command-line `markdownlint-cli2` run both read it, so the IDE and CLI stay in lock-step. Rules it deliberately disables (e.g. `MD013` line-length, `MD033` inline HTML) are **intentional** - do not "fix" them. Fix violations at the source rather than disabling rules. 2. **Spelling**: All spelling must be clean via the CSpell VS Code integration; words must be correctly spelled in **US English** (the repo-wide convention - see [AGENTS.md](./AGENTS.md)). Project-specific terms go in the workspace CSpell config. +3. **Spelling CI scope**: The enforced CI spell-check gate covers **`README.md` and `HISTORY.md` only** - these are the files every repo visitor sees, so they must be clean. It is deliberately **not** all `**/*.md`: repos carry many markdown files full of technical terms, and gating every one of them would mean endlessly padding `cspell.json` just to keep CI green. Broad, live spell-checking across any file (source, markdown, text) is the **cspell editor extension's** job, so typos still surface to whoever is editing. A repo owner **may** widen their own CI file list, but the template ships README + HISTORY as the default; keep every surface that runs cspell - the CI workflow and any local VS Code task or one-liner the repo has - on the same file list. The list is explicit (not a glob), so a repo that ships no `HISTORY.md` (e.g. one with no changelog) must drop it from all three surfaces and gate on `README.md` alone - cspell errors on a listed file that does not exist. Markdown *linting* (item 1) stays repo-wide `**/*.md` - it does not choke on technical terms. ## .NET From fb6cd8a9cdec10299b71f664ce6fabd9ab8255af Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Wed, 15 Jul 2026 22:00:25 -0700 Subject: [PATCH 13/19] Improve tool execution and failure logging (#845) Consolidate and correct media-tool execution logging: - Combine the split tool warning logs into a single line. - Capture failures on the stream each tool actually uses (ffmpeg/ffprobe/HandBrake/7-Zip -> stderr; mkvtoolnix/MediaInfo -> stdout), and include the operation context (via [CallerMemberName]) and the filename in the failure log. - Buffer mkvpropedit and 7-Zip executions (avoid unconsumed-pipe hangs). - Move Serilog to GlobalUsings; remove redundant per-call debug lines; delete the unused Execute overload. - HISTORY.md and AGENTS.md updated. Co-Authored-By: Claude Opus 4.8 (1M context) --- AGENTS.md | 12 +- HISTORY.md | 15 ++- PlexCleaner/Bitrate.cs | 1 - PlexCleaner/ConfigFileJsonSchema.cs | 1 - PlexCleaner/Convert.cs | 1 - PlexCleaner/ConvertOptions.cs | 1 - PlexCleaner/Extensions.cs | 2 - PlexCleaner/FfMpegIdetInfo.cs | 1 - PlexCleaner/FfMpegTool.cs | 15 ++- PlexCleaner/FfProbeTool.cs | 41 +++---- PlexCleaner/GitHubRelease.cs | 1 - PlexCleaner/GlobalUsings.cs | 1 + PlexCleaner/HandBrakeTool.cs | 3 +- PlexCleaner/IProcessPlugin.cs | 10 +- PlexCleaner/LoggerFactory.cs | 1 - PlexCleaner/MatroskaStructure.cs | 1 - PlexCleaner/MediaInfoTool.cs | 16 +-- PlexCleaner/MediaProps.cs | 1 - PlexCleaner/MediaTool.cs | 108 ++++++++---------- PlexCleaner/MediaToolInfo.cs | 2 - PlexCleaner/MkvMergeTool.cs | 28 +---- PlexCleaner/MkvPropEditTool.cs | 24 ++-- PlexCleaner/Monitor.cs | 2 - PlexCleaner/PluginLoader.cs | 1 - PlexCleaner/Process.cs | 1 - PlexCleaner/ProcessDriver.cs | 1 - PlexCleaner/ProcessFile.cs | 4 - PlexCleaner/ProcessOptions.cs | 1 - PlexCleaner/Program.cs | 1 - PlexCleaner/SevenZipTool.cs | 5 +- PlexCleaner/SidecarFile.cs | 1 - PlexCleaner/SidecarFileJsonSchema.cs | 1 - PlexCleaner/SubtitleProps.cs | 1 - PlexCleaner/TagMapSet.cs | 2 - PlexCleaner/ToolInfoJsonSchema.cs | 1 - PlexCleaner/Tools.cs | 6 +- PlexCleaner/ToolsOptions.cs | 1 - PlexCleaner/TrackProps.cs | 1 - PlexCleaner/VideoProps.cs | 1 - PlexCleanerTests/ToolFailureLogFormatTests.cs | 37 +++++- 40 files changed, 165 insertions(+), 189 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 2c9aa55a..82f6bad6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -198,11 +198,21 @@ Serilog log levels describe the **nature** of an event, applied uniformly across - A "modification" is a write to the **media file**, including in-place metadata edits (MkvPropEdit flags/language/title) and container remuxes/renames. Sidecar cache writes and the results file are bookkeeping, not media modifications - they are Debug/Information, not Warnings. - **The media-manipulation code itself does not emit Warning.** Doing a remux or re-encode is that code's job, not a warning. Only the decision to run it is the Warning. Do not sprinkle Warnings through `Convert`, the media-tool wrappers, or the worker methods. - **Information** - the high-level narrative of what the app is doing, readable end to end at the default level with no low-level mechanics: startup (banner, settings, tool versions), discovery (`Discovered N files`), batch lifecycle (`Starting {Command}, processing N files`, progress, `Completed`, the run summary), the per-file entry, read-only outcomes of note (skips), a worker **doing its job** (e.g. `Convert.ReMux` logging `Remux using MkvMerge`), and the intended output of read-only commands (`getmediainfo` / `getsidecarinfo` / `gettagmap` dumps). -- **Debug** - troubleshooting detail; *how* the work is done: raw tool invocations and command lines (`Executing MkvMerge : args`), read/probe mechanics (`Getting media info`, `Reading media info from sidecar`, temp files, packet probes), per-track structural dumps during normal processing, inspection sub-steps (verify, bitrate, idet counting), and sidecar cache bookkeeping. +- **Debug** - troubleshooting detail; *how* the work is done: raw tool invocations and command lines (`Executing MkvMerge : GetMediaPropsJson : args`, which carry the operation so a per-method "doing X" line is not needed), read/probe mechanics (`Reading media info from sidecar`, temp files, packet probes), per-track structural dumps during normal processing, inspection sub-steps (verify, bitrate, idet counting), and sidecar cache bookkeeping. - **Verbose** - very granular: filesystem-watcher events, per-packet/byte-level progress. The elevation trigger (Warning) must be preserved: keep exactly one decision-Warning per media modification, with the action at Information and the underlying tool at Debug. +### Tool execution and failure logging + +- **Always consume a tool's output.** A subprocess whose stdout/stderr is not read can deadlock once it fills the pipe buffer, so never run a tool without consuming its pipes: `MediaTool.Execute` buffers them (summarize when the output is huge), and `ExecuteStreamStdErr` streams stderr line by line for the unbounded `-f null` verify pass. `Execute`, its cancellation path, and `LogFailedResult` record the **operation** (the calling method, captured via `[CallerMemberName]`, rendered with `:l`) so a command line ties to its purpose in a parallel log without correlating separate lines. +- **Tools write errors to different streams.** ffmpeg, ffprobe, HandBrake, and 7-Zip use **stderr**; the mkvtoolnix tools (mkvmerge, mkvpropedit) write everything including errors to **stdout** (confirmed from the mkvtoolnix source - all output goes through the one stdout object) and override `GetErrorOutput` to it. MediaInfo also emits to stdout but keeps the stderr default; its errors are caught by the `LogFailedResult` fallback, which reads the other captured stream when the tool's declared stream is empty, so an error is never lost. +- **Do not add a per-method debug line that just restates the command about to run** (e.g. `Getting media info`); the `Executing {Tool} : {operation} : args` line from `Execute` already covers it. + +### Failure-handling philosophy + +An **expected, recoverable** failure escalates through the standard repair tiers (detect -> surgical -> remux -> re-encode -> fail); an **unexpected or logic** failure (e.g. tool output that will not parse) aborts the file and stays a hard error, so the bug surfaces and gets fixed rather than being masked by a fallback that silently mis-processes at scale. + ## Project Structure - **PlexCleaner** (`PlexCleaner/PlexCleaner.csproj`) diff --git a/HISTORY.md b/HISTORY.md index 4e74fa0e..9674a8bd 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -8,16 +8,23 @@ Utility to optimize media files for Direct Play in Plex, Emby, Jellyfin, etc. - Repair non-monotonic DTS muxer warnings losslessly instead of failing repair permanently. - `ffmpeg -f null` can exit `0` yet emit `Application provided invalid, non monotonically increasing dts to muxer` for files that may decode and play correctly. - The previous "any stderr means failure" rule promoted this muxer-interleaving artifact to a hard `VerifyFailed`/`RepairFailed`, and a re-encode could not fix it because Matroska stores no DTS and ffmpeg re-derives a non-monotonic timeline on read. - - Verify now classifies the decode diagnostics deterministically as clean, a timestamp-only failure, or a decode error; a timestamp-only failure is repaired losslessly when the break is demux-visible and otherwise stays reported, and everything else fails (fail-closed, so an unrecognized diagnostic fails as a decode error). + - Verify now classifies the decode diagnostics deterministically as clean, a timestamp-only failure, or a decode error; a timestamp-only failure is a repairable failure and everything else fails (fail-closed, so an unrecognized diagnostic fails as a decode error). - The classification streams the output line by line, so memory stays bounded even when a file emits a warning per packet ([#827](https://github.com/ptr727/PlexCleaner/issues/827)). - - Added a lossless timestamp repair as the first repair tier. - - When verification detects a demux-visible non-monotonic DTS on an audio stream, the audio packet timestamps are rewritten to be strictly monotonic using the `setts` bitstream filter with a stream copy (no re-encode), then re-verified. A video-stream DTS is not audio-repairable (a video `setts` would reorder B-frames) and stays reported. - - A regression gate compares the per-stream coded payload hash and the per-stream start and duration before and after, discarding the result unless every stream is byte-identical and no stream shifted beyond the A/V-sync tolerance, so the repair can neither alter the media nor drift the audio out of sync. The full re-encode repair remains for genuine decode corruption. + - Added a lossless timestamp repair as the first repair tier, escalating to remux and re-encode when it cannot apply. + - When verification detects a demux-visible non-monotonic DTS on an audio stream, the audio packet timestamps are rewritten to be strictly monotonic using the `setts` bitstream filter with a stream copy (no re-encode), then re-verified. + - A regression gate compares the per-stream coded payload hash and the per-stream start and duration before and after, discarding the result unless every stream is byte-identical and no stream shifted beyond the A/V-sync tolerance, so the lossless repair can neither alter the media nor drift the audio out of sync. + - A non-monotonic DTS the `setts` repair cannot fix - a video stream (where a `setts` would reorder B-frames), or a break visible only after decode - falls through to a remux and then a full re-encode that rebuilds the timestamps, matching the general detect -> surgical -> remux -> re-encode -> fail escalation, instead of stopping at `RepairFailed`. The re-encode tier also repairs genuine decode corruption. - Consolidated the bitrate and DTS packet analyses into a single `ffprobe -show_packets` pass, computing the per-second bitrate and the per-stream DTS monotonicity together instead of reading packets twice. - Switched closed caption detection to `ffprobe -analyze_frames -show_entries stream=closed_captions`, replacing the `movie=...[out0+subcc]` lavfi filter and its QuickScan snippet-remux workaround; QuickScan now bounds the scan with `-read_intervals`. - Added the `DtsTimestampRepair` example plugin. - It revisits files that a previous version marked `RepairFailed`, re-verifies them, clears the flag when the only problem was timestamps, and losslessly repairs the timestamps when the DTS is demux-visible. Not available in AOT builds. - Restricted `--testsnippets` to slow re-encode and deinterlace operations. Fast remux, stream-copy, and the lossless timestamp repair now always produce full output, so a repair or remux is validated on the whole file rather than an unrepresentative leading clip; a snippet had caused the timestamp-repair byte-identical gate to fail during testing. + - Hardened interlace detection against interleaved `idet` output. On a source with non-monotonic DTS, `ffmpeg -fflags +genpts` emits muxer warnings between the `idet` stat lines; the parser now matches each stat line independently instead of requiring a contiguous block, so a full-file scan no longer fails to parse and abort the file. The raw output is logged on a parse failure. + - Improved tool execution logging for troubleshooting. + - A tool failure now logs the tool's error output on a single line with the exit code, the operation, and the file name, instead of a bare exit code with the error text discarded or split across lines. + - The error text is read from the stream the tool writes to (stderr for the ffmpeg family, HandBrake, and 7-Zip; stdout for the mkvtoolnix tools), falling back to the other captured stream so output on an unexpected stream, such as MediaInfo's stdout, is never lost. The previously non-buffered mkvpropedit and 7-Zip executions now buffer their output, so their failures log the error text instead of nothing. + - The operation (the calling method, via `[CallerMemberName]`) is included in the execution, cancellation, and failure lines, so a command can be tied to its purpose in a parallel log without correlating separate lines. Redundant per-operation debug lines already covered by the command execution log were removed. + - Added per-file elapsed processing time to the `ProcessFiles` result line, formatted consistently with the run total. - Version 3.20: - Switched tool downloads and the application version check to the resilient HTTP client in `ptr727.Utilities` (retry with backoff and a circuit breaker via `Microsoft.Extensions.Http.Resilience`), replacing the plain `HttpClient`. - Enabled closed caption removal for H.265/HEVC video: the SEI NAL unit lookup keyed on `h265` never matched FFprobe's `hevc` codec name, so HEVC files were incorrectly reported as an "Unsupported video format for Closed Captions removal". HEVC video (excluding HDR10 and HDR10+ content, which remains guarded) is now cleaned using the `filter_units=remove_types=39` bitstream filter, same as H.264 and MPEG-2. diff --git a/PlexCleaner/Bitrate.cs b/PlexCleaner/Bitrate.cs index a1d47668..754b47e9 100644 --- a/PlexCleaner/Bitrate.cs +++ b/PlexCleaner/Bitrate.cs @@ -1,6 +1,5 @@ using System.Diagnostics; using ptr727.Utilities; -using Serilog; namespace PlexCleaner; diff --git a/PlexCleaner/ConfigFileJsonSchema.cs b/PlexCleaner/ConfigFileJsonSchema.cs index 09f3f7b1..55144ffb 100644 --- a/PlexCleaner/ConfigFileJsonSchema.cs +++ b/PlexCleaner/ConfigFileJsonSchema.cs @@ -11,7 +11,6 @@ using System.Text.Json.Nodes; using System.Text.Json.Schema; using System.Text.Json.Serialization; -using Serilog; namespace PlexCleaner; diff --git a/PlexCleaner/Convert.cs b/PlexCleaner/Convert.cs index a9043c34..e2edb3db 100644 --- a/PlexCleaner/Convert.cs +++ b/PlexCleaner/Convert.cs @@ -1,5 +1,4 @@ using System.Diagnostics; -using Serilog; namespace PlexCleaner; diff --git a/PlexCleaner/ConvertOptions.cs b/PlexCleaner/ConvertOptions.cs index b2e26089..c23071b6 100644 --- a/PlexCleaner/ConvertOptions.cs +++ b/PlexCleaner/ConvertOptions.cs @@ -1,5 +1,4 @@ using System.Text.Json.Serialization; -using Serilog; namespace PlexCleaner; diff --git a/PlexCleaner/Extensions.cs b/PlexCleaner/Extensions.cs index bb4e1c79..45f4fed0 100644 --- a/PlexCleaner/Extensions.cs +++ b/PlexCleaner/Extensions.cs @@ -1,5 +1,3 @@ -using Serilog; - namespace PlexCleaner; public static class Extensions diff --git a/PlexCleaner/FfMpegIdetInfo.cs b/PlexCleaner/FfMpegIdetInfo.cs index 822c697f..a16dbe25 100644 --- a/PlexCleaner/FfMpegIdetInfo.cs +++ b/PlexCleaner/FfMpegIdetInfo.cs @@ -1,7 +1,6 @@ using System.Diagnostics; using System.Globalization; using System.Text.RegularExpressions; -using Serilog; namespace PlexCleaner; diff --git a/PlexCleaner/FfMpegTool.cs b/PlexCleaner/FfMpegTool.cs index 9ec8de06..9766e4af 100644 --- a/PlexCleaner/FfMpegTool.cs +++ b/PlexCleaner/FfMpegTool.cs @@ -4,7 +4,6 @@ using System.Text.RegularExpressions; using CliWrap; using CliWrap.Buffered; -using Serilog; // https://ffmpeg.org/ffmpeg.html @@ -221,7 +220,7 @@ public bool ReMuxToFormat(string inputName, string outputName, string format) // Execute command return Execute(command, true, true, out BufferedCommandResult result) - && (result.ExitCode == 0 || LogFailedResult(result)); + && (result.ExitCode == 0 || LogFailedResult(result, inputName)); } private static void CreateTrackArgs( @@ -321,7 +320,7 @@ string outputName // Execute command return Execute(command, true, true, out BufferedCommandResult result) - && (result.ExitCode == 0 || LogFailedResult(result)); + && (result.ExitCode == 0 || LogFailedResult(result, inputName)); } public bool ConvertToMkv(string inputName, string outputName) @@ -349,7 +348,7 @@ public bool ConvertToMkv(string inputName, string outputName) // Execute command return Execute(command, true, true, out BufferedCommandResult result) - && (result.ExitCode == 0 || LogFailedResult(result)); + && (result.ExitCode == 0 || LogFailedResult(result, inputName)); } public bool SetTimestamps(string inputName, string outputName) @@ -382,7 +381,7 @@ public bool SetTimestamps(string inputName, string outputName) // Execute command return Execute(command, true, true, out BufferedCommandResult result) - && (result.ExitCode == 0 || LogFailedResult(result)); + && (result.ExitCode == 0 || LogFailedResult(result, inputName)); } public bool GetStreamHashes(string fileName, out Dictionary streamHashes) @@ -406,7 +405,7 @@ public bool GetStreamHashes(string fileName, out Dictionary streamH } if (result.ExitCode != 0) { - return LogFailedResult(result); + return LogFailedResult(result, fileName); } // Parse lines of the form "index,type,md5=value" @@ -455,7 +454,7 @@ public bool RemoveNalUnits(string inputName, int nalUnit, string outputName) // Execute command return Execute(command, true, true, out BufferedCommandResult result) - && (result.ExitCode == 0 || LogFailedResult(result)); + && (result.ExitCode == 0 || LogFailedResult(result, inputName)); } public bool GetIdetText(string fileName, out string text) @@ -481,7 +480,7 @@ public bool GetIdetText(string fileName, out string text) return false; } text = result.StandardError.Trim(); - return result.ExitCode == 0 || LogFailedResult(result); + return result.ExitCode == 0 || LogFailedResult(result, fileName); } [GeneratedRegex( diff --git a/PlexCleaner/FfProbeTool.cs b/PlexCleaner/FfProbeTool.cs index 51c39599..67e6e84d 100644 --- a/PlexCleaner/FfProbeTool.cs +++ b/PlexCleaner/FfProbeTool.cs @@ -1,10 +1,10 @@ +using System.Runtime.CompilerServices; using System.Text; using System.Text.Json; using System.Text.Json.Serialization; using System.Text.Json.Stream; using CliWrap; using CliWrap.Buffered; -using Serilog; // https://ffmpeg.org/ffprobe.html @@ -56,13 +56,15 @@ protected override bool GetLatestVersionWindows(out MediaToolInfo mediaToolInfo) public bool GetPackets( Command command, Func packetFunc, - out string error + out string error, + [CallerMemberName] string operation = "" ) { // Wrap async function in a task (bool result, string error) result = GetPacketsAsync( command, - async packet => await Task.FromResult(packetFunc(packet)) + async packet => await Task.FromResult(packetFunc(packet)), + operation ) .GetAwaiter() .GetResult(); @@ -72,7 +74,8 @@ out string error public async Task<(bool result, string error)> GetPacketsAsync( Command command, - Func> packetFunc + Func> packetFunc, + [CallerMemberName] string operation = "" ) { int processId = -1; @@ -165,8 +168,9 @@ out string error .ExecuteAsync(CancellationToken.None, Program.CancelToken()); processId = task.ProcessId; Log.Debug( - "Executing {ToolType} : ProcessId: {ProcessId}, Arguments: {Arguments}", + "Executing {ToolType} : {Operation:l} : ProcessId: {ProcessId}, Arguments: {Arguments}", GetToolType(), + operation, processId, command.Arguments ); @@ -178,8 +182,9 @@ out string error catch (OperationCanceledException) { Log.Error( - "Cancelled execution of {ToolType} : ProcessId: {ProcessId}, Arguments: {Arguments}", + "Cancelled execution of {ToolType} : {Operation:l} : ProcessId: {ProcessId}, Arguments: {Arguments}", GetToolType(), + operation, processId, command.Arguments ); @@ -212,15 +217,13 @@ public bool GetClosedCaptions(string fileName, out bool hasClosedCaptions) .Build(); // Execute command - Log.Debug("Getting closed caption info : {FileName}", fileName); if (!Execute(command, false, true, out BufferedCommandResult result)) { return false; } if (result.ExitCode != 0) { - Log.Error("Failed to get closed caption info : {FileName}", fileName); - return LogFailedResult(result); + return LogFailedResult(result, fileName); } // Any video stream reporting closed captions, FromJson throws on malformed output @@ -256,15 +259,13 @@ public bool GetStreamTimings( .Build(); // Execute command - Log.Debug("Getting stream timings : {FileName}", fileName); if (!Execute(command, false, true, out BufferedCommandResult result)) { return false; } if (result.ExitCode != 0) { - Log.Error("Failed to get stream timings : {FileName}", fileName); - return LogFailedResult(result); + return LogFailedResult(result, fileName); } // FromJson throws on malformed output @@ -303,7 +304,6 @@ bool quickScan .Build(); // Get packet list - Log.Debug("Getting analysis packets : {FileName}", fileName); if (!GetPackets(command, packetFunc, out string error)) { Log.Error("Failed to get analysis packets : {FileName}", fileName); @@ -334,28 +334,23 @@ public bool GetMediaPropsJson(string fileName, out string json) .Build(); // Execute command - Log.Debug("{ToolType} : Getting media info : {FileName}", GetToolType(), fileName); if (!Execute(command, false, true, out BufferedCommandResult result)) { return false; } if (result.ExitCode != 0) { - Log.Error( - "{ToolType} : Failed to get media info : {FileName}", - GetToolType(), - fileName - ); - return LogFailedResult(result); + return LogFailedResult(result, fileName); } - if (result.StandardError.Length > 0) + string warning = CleanForLog(result.StandardError.Trim()); + if (!string.IsNullOrEmpty(warning)) { Log.Warning( - "{ToolType} : Warning getting media info : {FileName}", + "{ToolType} : Warning getting media info : {Warning} : {FileName}", GetToolType(), + warning, fileName ); - Log.Warning("{ToolType} : {Warning}", GetToolType(), result.StandardError.Trim()); } // Get JSON output diff --git a/PlexCleaner/GitHubRelease.cs b/PlexCleaner/GitHubRelease.cs index 5b420384..0461a40c 100644 --- a/PlexCleaner/GitHubRelease.cs +++ b/PlexCleaner/GitHubRelease.cs @@ -1,6 +1,5 @@ using System.Text.Json.Nodes; using ptr727.Utilities; -using Serilog; namespace PlexCleaner; diff --git a/PlexCleaner/GlobalUsings.cs b/PlexCleaner/GlobalUsings.cs index d9c83552..31d7a820 100644 --- a/PlexCleaner/GlobalUsings.cs +++ b/PlexCleaner/GlobalUsings.cs @@ -1,2 +1,3 @@ +global using Serilog; global using ConfigFileJsonSchema = PlexCleaner.ConfigFileJsonSchema4; global using SidecarFileJsonSchema = PlexCleaner.SidecarFileJsonSchema5; diff --git a/PlexCleaner/HandBrakeTool.cs b/PlexCleaner/HandBrakeTool.cs index ffd133ff..a22c2d53 100644 --- a/PlexCleaner/HandBrakeTool.cs +++ b/PlexCleaner/HandBrakeTool.cs @@ -2,7 +2,6 @@ using System.Text.RegularExpressions; using CliWrap; using CliWrap.Buffered; -using Serilog; // https://handbrake.fr/docs/en/latest/cli/command-line-reference.html @@ -124,7 +123,7 @@ bool deInterlace // Execute command return Execute(command, true, true, out BufferedCommandResult result) - && (result.ExitCode == 0 || LogFailedResult(result)); + && (result.ExitCode == 0 || LogFailedResult(result, inputName)); } [GeneratedRegex( diff --git a/PlexCleaner/IProcessPlugin.cs b/PlexCleaner/IProcessPlugin.cs index 2b8e1f96..b400e7f3 100644 --- a/PlexCleaner/IProcessPlugin.cs +++ b/PlexCleaner/IProcessPlugin.cs @@ -1,3 +1,5 @@ +using System.Diagnostics.CodeAnalysis; + namespace PlexCleaner; public static class PluginApi @@ -16,7 +18,13 @@ public interface IPluginHost string OperatingSystem { get; } string Runtime { get; } - // Plugin log events flow to the host sinks and end-of-run summary + // Plugin log events flow to the host sinks and end-of-run summary. Fully qualified so plugin authors + // are not left guessing between Serilog.ILogger and Microsoft.Extensions.Logging.ILogger + [SuppressMessage( + "Style", + "IDE0001:Simplify Names", + Justification = "Disambiguate the public plugin logger type" + )] Serilog.ILogger Logger { get; } } diff --git a/PlexCleaner/LoggerFactory.cs b/PlexCleaner/LoggerFactory.cs index 67577846..9b4566e8 100644 --- a/PlexCleaner/LoggerFactory.cs +++ b/PlexCleaner/LoggerFactory.cs @@ -1,5 +1,4 @@ using System.Globalization; -using Serilog; using Serilog.Debugging; using Serilog.Events; using Serilog.Sinks.SystemConsole.Themes; diff --git a/PlexCleaner/MatroskaStructure.cs b/PlexCleaner/MatroskaStructure.cs index 752099a3..d1b586c8 100644 --- a/PlexCleaner/MatroskaStructure.cs +++ b/PlexCleaner/MatroskaStructure.cs @@ -1,5 +1,4 @@ using NEbml.Core; -using Serilog; namespace PlexCleaner; diff --git a/PlexCleaner/MediaInfoTool.cs b/PlexCleaner/MediaInfoTool.cs index 031ae9c0..8f69b167 100644 --- a/PlexCleaner/MediaInfoTool.cs +++ b/PlexCleaner/MediaInfoTool.cs @@ -3,7 +3,6 @@ using System.Text.RegularExpressions; using CliWrap; using CliWrap.Buffered; -using Serilog; // http://manpages.ubuntu.com/manpages/zesty/man1/mediainfo.1.html @@ -111,28 +110,23 @@ public bool GetMediaPropsJson(string fileName, out string json) .Build(); // Execute command - Log.Debug("Getting media info : {FileName}", fileName); if (!Execute(command, false, true, out BufferedCommandResult result)) { return false; } if (result.ExitCode != 0) { - Log.Error( - "{ToolType} : Failed to get media info : {FileName}", - GetToolType(), - fileName - ); - return LogFailedResult(result); + return LogFailedResult(result, fileName); } - if (result.StandardError.Length > 0) + string warning = CleanForLog(result.StandardError.Trim()); + if (!string.IsNullOrEmpty(warning)) { Log.Warning( - "{ToolType} : Warning getting media info : {FileName}", + "{ToolType} : Warning getting media info : {Warning} : {FileName}", GetToolType(), + warning, fileName ); - Log.Warning("{ToolType} : {Warning}", GetToolType(), result.StandardError.Trim()); } // Get JSON output diff --git a/PlexCleaner/MediaProps.cs b/PlexCleaner/MediaProps.cs index 3e217e2e..cd556f38 100644 --- a/PlexCleaner/MediaProps.cs +++ b/PlexCleaner/MediaProps.cs @@ -1,5 +1,4 @@ using System.Diagnostics; -using Serilog; using Serilog.Events; namespace PlexCleaner; diff --git a/PlexCleaner/MediaTool.cs b/PlexCleaner/MediaTool.cs index d75df70e..0eb9c7a3 100644 --- a/PlexCleaner/MediaTool.cs +++ b/PlexCleaner/MediaTool.cs @@ -1,8 +1,8 @@ +using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Text; using CliWrap; using CliWrap.Buffered; -using Serilog; namespace PlexCleaner; @@ -94,70 +94,54 @@ protected bool GetLatestGitHubRelease(string repo, out string version) return GitHubRelease.GetLatestRelease(repo, out version); } - public bool Execute(Command command, out CommandResult commandResult) - { - commandResult = null!; - int processId = -1; - try - { - CommandTask task = command - .WithValidation(CommandResultValidation.None) - .ExecuteAsync(CancellationToken.None, Program.CancelToken()); - processId = task.ProcessId; - Log.Debug( - "Executing {ToolType} : ProcessId: {ProcessId}, Arguments: {Arguments}", - GetToolType(), - processId, - command.Arguments - ); - - commandResult = task.Task.GetAwaiter().GetResult(); - return task.Task.IsCompletedSuccessfully; - } - catch (OperationCanceledException) - { - Log.Error( - "Cancelled execution of {ToolType} : ProcessId: {ProcessId}, Arguments: {Arguments}", - GetToolType(), - processId, - command.Arguments - ); - return false; - } - catch (Exception e) when (Log.Logger.LogAndHandle(e)) - { - return false; - } - } - - public bool Execute(Command command, out BufferedCommandResult bufferedCommandResult) => - Execute(command, false, false, out bufferedCommandResult); + public bool Execute( + Command command, + out BufferedCommandResult bufferedCommandResult, + [CallerMemberName] string operation = "" + ) => Execute(command, false, false, out bufferedCommandResult, operation); - // Stream carrying tool error text; stderr by default, stdout for MkvMerge + // Stream carrying tool error text; stderr by default, stdout for the mkvtoolnix tools protected virtual string GetErrorOutput(BufferedCommandResult result) => result.StandardError; - protected bool LogFailedResult(BufferedCommandResult result) + protected bool LogFailedResult( + BufferedCommandResult result, + string fileName, + [CallerMemberName] string operation = "" + ) { // ffmpeg can exit 0 yet report a fatal error on stderr (see FfMpegTool), so failures may carry // an error summary. Log the summary as its own value with the " : " separator in the template. // Folding the separator into a quoted string value instead puts the quote right after the exit // code, rendering: ExitCode: 0" : ... instead of the correct ExitCode: 0 : "...". - string summary = CleanForLog(Summarize(GetErrorOutput(result).Trim())); + // Prefer the stream the tool writes errors to (GetErrorOutput); if it is empty, fall back to the + // other captured stream so an error on an unexpected stream is still logged. + string stdErr = result.StandardError.Trim(); + string stdOut = result.StandardOutput.Trim(); + string primary = GetErrorOutput(result).Trim(); + string error = + !string.IsNullOrEmpty(primary) ? primary + : !string.IsNullOrEmpty(stdErr) ? stdErr + : stdOut; + string summary = CleanForLog(Summarize(error)); if (string.IsNullOrEmpty(summary)) { Log.Error( - "Failed execution of {ToolType} : ExitCode: {ExitCode}", + "Failed execution of {ToolType} : {Operation:l} : ExitCode: {ExitCode} : {FileName}", GetToolType(), - result.ExitCode + operation, + result.ExitCode, + fileName ); } else { Log.Error( - "Failed execution of {ToolType} : ExitCode: {ExitCode} : {Error}", + "Failed execution of {ToolType} : {Operation:l} : ExitCode: {ExitCode} : {Error} : {FileName}", GetToolType(), + operation, result.ExitCode, - summary + summary, + fileName ); } return false; @@ -179,21 +163,12 @@ protected static string CleanForLog(string text) return builder.ToString().Trim(); } - protected bool LogFailedResult(CommandResult result) - { - Log.Error( - "Failed execution of {ToolType} : ExitCode: {ExitCode}", - GetToolType(), - result.ExitCode - ); - return false; - } - public bool Execute( Command command, bool stdOutSummary, bool stdErrSummary, - out BufferedCommandResult bufferedCommandResult + out BufferedCommandResult bufferedCommandResult, + [CallerMemberName] string operation = "" ) { bufferedCommandResult = null!; @@ -216,8 +191,9 @@ out BufferedCommandResult bufferedCommandResult .ExecuteAsync(CancellationToken.None, Program.CancelToken()); processId = task.ProcessId; Log.Debug( - "Executing {ToolType} : ProcessId: {ProcessId}, Arguments: {Arguments}", + "Executing {ToolType} : {Operation:l} : ProcessId: {ProcessId}, Arguments: {Arguments}", GetToolType(), + operation, processId, command.Arguments ); @@ -235,8 +211,9 @@ out BufferedCommandResult bufferedCommandResult catch (OperationCanceledException) { Log.Error( - "Cancelled execution of {ToolType} : ProcessId: {ProcessId}, Arguments: {Arguments}", + "Cancelled execution of {ToolType} : {Operation:l} : ProcessId: {ProcessId}, Arguments: {Arguments}", GetToolType(), + operation, processId, command.Arguments ); @@ -248,7 +225,12 @@ out BufferedCommandResult bufferedCommandResult } } - public bool ExecuteStreamStdErr(Command command, Action lineAction, out int exitCode) + public bool ExecuteStreamStdErr( + Command command, + Action lineAction, + out int exitCode, + [CallerMemberName] string operation = "" + ) { exitCode = -1; int processId = -1; @@ -277,8 +259,9 @@ public bool ExecuteStreamStdErr(Command command, Action lineAction, out .ExecuteAsync(CancellationToken.None, Program.CancelToken()); processId = task.ProcessId; Log.Debug( - "Executing {ToolType} : ProcessId: {ProcessId}, Arguments: {Arguments}", + "Executing {ToolType} : {Operation:l} : ProcessId: {ProcessId}, Arguments: {Arguments}", GetToolType(), + operation, processId, command.Arguments ); @@ -290,8 +273,9 @@ public bool ExecuteStreamStdErr(Command command, Action lineAction, out catch (OperationCanceledException) { Log.Error( - "Cancelled execution of {ToolType} : ProcessId: {ProcessId}, Arguments: {Arguments}", + "Cancelled execution of {ToolType} : {Operation:l} : ProcessId: {ProcessId}, Arguments: {Arguments}", GetToolType(), + operation, processId, command.Arguments ); diff --git a/PlexCleaner/MediaToolInfo.cs b/PlexCleaner/MediaToolInfo.cs index 9dfe4abc..29c6d260 100644 --- a/PlexCleaner/MediaToolInfo.cs +++ b/PlexCleaner/MediaToolInfo.cs @@ -1,5 +1,3 @@ -using Serilog; - namespace PlexCleaner; public class MediaToolInfo diff --git a/PlexCleaner/MkvMergeTool.cs b/PlexCleaner/MkvMergeTool.cs index 4223b4e4..6c61801a 100644 --- a/PlexCleaner/MkvMergeTool.cs +++ b/PlexCleaner/MkvMergeTool.cs @@ -3,7 +3,6 @@ using CliWrap; using CliWrap.Buffered; using ptr727.Utilities; -using Serilog; // https://mkvtoolnix.download/doc/mkvmerge.html @@ -120,30 +119,15 @@ public bool GetMediaPropsJson(string fileName, out string json) .Build(); // Execute command - Log.Debug("Getting media info : {FileName}", fileName); if (!Execute(command, false, true, out BufferedCommandResult result)) { return false; } if (result.ExitCode != 0) { - Log.Error( - "{ToolType} : Failed to get media info : {FileName}", - GetToolType(), - fileName - ); - return LogFailedResult(result); - } - if (result.StandardError.Length > 0) - { - // TODO: This probably never gets hit due to mkvmerge not using stderr - Log.Warning( - "{ToolType} : Warning getting media info : {FileName}", - GetToolType(), - fileName - ); - Log.Warning("{ToolType} : {Warning}", GetToolType(), result.StandardError.Trim()); + return LogFailedResult(result, fileName); } + // Ignore "if (result.StandardError.Length > 0)" pattern, mkv tools only emit to stdout // Get JSON from stdout json = result.StandardOutput; @@ -267,7 +251,7 @@ string outputName // Execute command return Execute(command, true, true, out BufferedCommandResult result) - && (result.ExitCode is 0 or 1 || LogFailedResult(result)); + && (result.ExitCode is 0 or 1 || LogFailedResult(result, inputName)); } public bool ReMuxToMkv(string inputName, string outputName) @@ -284,7 +268,7 @@ public bool ReMuxToMkv(string inputName, string outputName) // Execute command return Execute(command, true, true, out BufferedCommandResult result) - && (result.ExitCode is 0 or 1 || LogFailedResult(result)); + && (result.ExitCode is 0 or 1 || LogFailedResult(result, inputName)); } public bool RemoveSubtitles(string inputName, string outputName) @@ -301,7 +285,7 @@ public bool RemoveSubtitles(string inputName, string outputName) // Execute command return Execute(command, true, true, out BufferedCommandResult result) - && (result.ExitCode is 0 or 1 || LogFailedResult(result)); + && (result.ExitCode is 0 or 1 || LogFailedResult(result, inputName)); } public bool MergeToMkv( @@ -335,7 +319,7 @@ string outputName // Execute command return Execute(command, true, true, out BufferedCommandResult result) - && (result.ExitCode is 0 or 1 || LogFailedResult(result)); + && (result.ExitCode is 0 or 1 || LogFailedResult(result, sourceOne)); } [GeneratedRegex( diff --git a/PlexCleaner/MkvPropEditTool.cs b/PlexCleaner/MkvPropEditTool.cs index bf88362b..987203e4 100644 --- a/PlexCleaner/MkvPropEditTool.cs +++ b/PlexCleaner/MkvPropEditTool.cs @@ -25,6 +25,10 @@ public class Tool : MediaTool public IGlobalOptions GetBuilder() => Builder.Create(GetToolPath()); + // mkvpropedit, like all mkvtoolnix tools, writes errors to stdout, not stderr + protected override string GetErrorOutput(BufferedCommandResult result) => + result.StandardOutput; + public override bool GetInstalledVersion(out MediaToolInfo mediaToolInfo) { // Get version info @@ -70,8 +74,8 @@ public bool SetTrackLanguage( .Build(); // Execute command - return Execute(command, out CommandResult result) - && (result.ExitCode is 0 || LogFailedResult(result)); + return Execute(command, out BufferedCommandResult result) + && (result.ExitCode is 0 || LogFailedResult(result, fileName)); } public bool SetTrackFlags(string fileName, MediaProps mediaProps) @@ -101,8 +105,8 @@ public bool SetTrackFlags(string fileName, MediaProps mediaProps) .Build(); // Execute command - return Execute(command, out CommandResult result) - && (result.ExitCode is 0 || LogFailedResult(result)); + return Execute(command, out BufferedCommandResult result) + && (result.ExitCode is 0 || LogFailedResult(result, fileName)); } public bool ClearDefaultFlags(string fileName, IEnumerable trackList) @@ -130,8 +134,8 @@ public bool ClearDefaultFlags(string fileName, IEnumerable trackList .Build(); // Execute command - return Execute(command, out CommandResult result) - && (result.ExitCode is 0 || LogFailedResult(result)); + return Execute(command, out BufferedCommandResult result) + && (result.ExitCode is 0 || LogFailedResult(result, fileName)); } public bool ClearTags(string fileName, MediaProps mediaProps) @@ -166,8 +170,8 @@ public bool ClearTags(string fileName, MediaProps mediaProps) .Build(); // Execute command - return Execute(command, out CommandResult result) - && (result.ExitCode is 0 || LogFailedResult(result)); + return Execute(command, out BufferedCommandResult result) + && (result.ExitCode is 0 || LogFailedResult(result, fileName)); } public bool ClearAttachments(string fileName, MediaProps mediaProps) @@ -194,8 +198,8 @@ public bool ClearAttachments(string fileName, MediaProps mediaProps) .Build(); // Execute command - return Execute(command, out CommandResult result) - && (result.ExitCode is 0 || LogFailedResult(result)); + return Execute(command, out BufferedCommandResult result) + && (result.ExitCode is 0 || LogFailedResult(result, fileName)); } } } diff --git a/PlexCleaner/Monitor.cs b/PlexCleaner/Monitor.cs index 6a4efb69..c2f97a32 100644 --- a/PlexCleaner/Monitor.cs +++ b/PlexCleaner/Monitor.cs @@ -1,5 +1,3 @@ -using Serilog; - namespace PlexCleaner; public class Monitor diff --git a/PlexCleaner/PluginLoader.cs b/PlexCleaner/PluginLoader.cs index 2544ee00..58458845 100644 --- a/PlexCleaner/PluginLoader.cs +++ b/PlexCleaner/PluginLoader.cs @@ -3,7 +3,6 @@ using System.Reflection; using System.Runtime.InteropServices; using System.Runtime.Loader; -using Serilog; namespace PlexCleaner; diff --git a/PlexCleaner/Process.cs b/PlexCleaner/Process.cs index 69994586..e35919a7 100644 --- a/PlexCleaner/Process.cs +++ b/PlexCleaner/Process.cs @@ -1,5 +1,4 @@ using System.Diagnostics; -using Serilog; namespace PlexCleaner; diff --git a/PlexCleaner/ProcessDriver.cs b/PlexCleaner/ProcessDriver.cs index b4545247..24c0f6fe 100644 --- a/PlexCleaner/ProcessDriver.cs +++ b/PlexCleaner/ProcessDriver.cs @@ -1,6 +1,5 @@ using System.Collections.Concurrent; using System.Diagnostics; -using Serilog; namespace PlexCleaner; diff --git a/PlexCleaner/ProcessFile.cs b/PlexCleaner/ProcessFile.cs index e2926d63..83b51913 100644 --- a/PlexCleaner/ProcessFile.cs +++ b/PlexCleaner/ProcessFile.cs @@ -1,5 +1,4 @@ using System.Diagnostics; -using Serilog; using Serilog.Events; namespace PlexCleaner; @@ -1760,7 +1759,6 @@ public bool Verify(bool conditional, out bool canRepair) public static VerifyResult VerifyMediaStreams(FileInfo fileInfo) { // Verify - Log.Debug("Verifying media streams : {FileName}", fileInfo.FullName); VerifyResult verifyResult = Tools.FfMpeg.VerifyMedia(fileInfo.FullName); // Log the classified outcome so a failure is diagnosable, unless it was a cancellation @@ -1920,7 +1918,6 @@ private bool VerifyBitrate() // https://en.wikipedia.org/wiki/YIFY // Calculate bitrate - Log.Debug("Calculating bitrate info : {FileName}", FileInfo.FullName); if (!GetBitrateInfo(out BitrateInfo? bitrateInfo) || bitrateInfo == null) { // Error @@ -2579,7 +2576,6 @@ out DtsInfo? dtsInfo private bool GetIdetInfo(out FfMpegIdetInfo? idetInfo) { // Count the frame types using the idet filter - Log.Debug("Counting interlaced frames : {FileName}", FileInfo.FullName); if (!FfMpegIdetInfo.GetIdetInfo(FileInfo.FullName, out idetInfo) || idetInfo == null) { // Cancel requested diff --git a/PlexCleaner/ProcessOptions.cs b/PlexCleaner/ProcessOptions.cs index a141c9ee..9e1fd05d 100644 --- a/PlexCleaner/ProcessOptions.cs +++ b/PlexCleaner/ProcessOptions.cs @@ -1,6 +1,5 @@ using System.Text.Json.Serialization; using System.Text.RegularExpressions; -using Serilog; namespace PlexCleaner; diff --git a/PlexCleaner/Program.cs b/PlexCleaner/Program.cs index ede7475b..e22b31fa 100644 --- a/PlexCleaner/Program.cs +++ b/PlexCleaner/Program.cs @@ -1,7 +1,6 @@ using System.Diagnostics; using System.Runtime.InteropServices; using ptr727.Utilities; -using Serilog; namespace PlexCleaner; diff --git a/PlexCleaner/SevenZipTool.cs b/PlexCleaner/SevenZipTool.cs index 5f6f9113..4f60f1fa 100644 --- a/PlexCleaner/SevenZipTool.cs +++ b/PlexCleaner/SevenZipTool.cs @@ -3,7 +3,6 @@ using System.Text.RegularExpressions; using CliWrap; using CliWrap.Buffered; -using Serilog; // 7za [...] [...] [<@listfiles...>] @@ -137,8 +136,8 @@ public bool UnZip(IGlobalOptions options, string inputFile, string outputFolder) .Build(); // Execute command - return Execute(command, out CommandResult result) - && (result.ExitCode == 0 || LogFailedResult(result)); + return Execute(command, out BufferedCommandResult result) + && (result.ExitCode == 0 || LogFailedResult(result, inputFile)); } public bool BootstrapDownload() diff --git a/PlexCleaner/SidecarFile.cs b/PlexCleaner/SidecarFile.cs index 9e34faa4..0a01ca1a 100644 --- a/PlexCleaner/SidecarFile.cs +++ b/PlexCleaner/SidecarFile.cs @@ -2,7 +2,6 @@ using System.Diagnostics; using System.Security.Cryptography; using ptr727.Utilities; -using Serilog; using Serilog.Events; namespace PlexCleaner; diff --git a/PlexCleaner/SidecarFileJsonSchema.cs b/PlexCleaner/SidecarFileJsonSchema.cs index 0ccd36c3..490319e0 100644 --- a/PlexCleaner/SidecarFileJsonSchema.cs +++ b/PlexCleaner/SidecarFileJsonSchema.cs @@ -3,7 +3,6 @@ using System.Text.Json; using System.Text.Json.Serialization; using ptr727.Utilities; -using Serilog; namespace PlexCleaner; diff --git a/PlexCleaner/SubtitleProps.cs b/PlexCleaner/SubtitleProps.cs index e8dbbb38..57b77f4f 100644 --- a/PlexCleaner/SubtitleProps.cs +++ b/PlexCleaner/SubtitleProps.cs @@ -1,5 +1,4 @@ using System.Globalization; -using Serilog; namespace PlexCleaner; diff --git a/PlexCleaner/TagMapSet.cs b/PlexCleaner/TagMapSet.cs index f300a643..2e6dc84f 100644 --- a/PlexCleaner/TagMapSet.cs +++ b/PlexCleaner/TagMapSet.cs @@ -1,5 +1,3 @@ -using Serilog; - namespace PlexCleaner; public class TagMapSet diff --git a/PlexCleaner/ToolInfoJsonSchema.cs b/PlexCleaner/ToolInfoJsonSchema.cs index 902420a4..aca3c18b 100644 --- a/PlexCleaner/ToolInfoJsonSchema.cs +++ b/PlexCleaner/ToolInfoJsonSchema.cs @@ -1,7 +1,6 @@ using System.ComponentModel; using System.Text.Json; using System.Text.Json.Serialization; -using Serilog; namespace PlexCleaner; diff --git a/PlexCleaner/Tools.cs b/PlexCleaner/Tools.cs index 249e2f74..79bb1e73 100644 --- a/PlexCleaner/Tools.cs +++ b/PlexCleaner/Tools.cs @@ -1,7 +1,6 @@ using System.Diagnostics; using System.Runtime.InteropServices; using ptr727.Utilities; -using Serilog; namespace PlexCleaner; @@ -30,8 +29,9 @@ public static bool VerifyTools() && !Program.Config.ToolsOptions.UseSystem ) { - Log.Warning("Folder tools are not supported on Linux"); - Log.Warning("Set 'ToolsOptions:UseSystem' to 'true' on Linux"); + Log.Warning( + "Folder tools are not supported on Linux, forcing 'ToolsOptions:UseSystem' to 'true'" + ); Program.Config.ToolsOptions.UseSystem = true; } diff --git a/PlexCleaner/ToolsOptions.cs b/PlexCleaner/ToolsOptions.cs index 6c16c273..78a8355c 100644 --- a/PlexCleaner/ToolsOptions.cs +++ b/PlexCleaner/ToolsOptions.cs @@ -1,6 +1,5 @@ using System.Runtime.InteropServices; using System.Text.Json.Serialization; -using Serilog; namespace PlexCleaner; diff --git a/PlexCleaner/TrackProps.cs b/PlexCleaner/TrackProps.cs index 95d2929b..b7cd8a5c 100644 --- a/PlexCleaner/TrackProps.cs +++ b/PlexCleaner/TrackProps.cs @@ -1,6 +1,5 @@ using System.Diagnostics; using System.Globalization; -using Serilog; using Serilog.Events; namespace PlexCleaner; diff --git a/PlexCleaner/VideoProps.cs b/PlexCleaner/VideoProps.cs index 43928ddc..4ac1f424 100644 --- a/PlexCleaner/VideoProps.cs +++ b/PlexCleaner/VideoProps.cs @@ -1,4 +1,3 @@ -using Serilog; using Serilog.Events; // TODO: Find a better way to create profile levels diff --git a/PlexCleanerTests/ToolFailureLogFormatTests.cs b/PlexCleanerTests/ToolFailureLogFormatTests.cs index 5f9c3a3c..88a6a87a 100644 --- a/PlexCleanerTests/ToolFailureLogFormatTests.cs +++ b/PlexCleanerTests/ToolFailureLogFormatTests.cs @@ -47,13 +47,23 @@ protected override bool GetLatestVersionWindows(out MediaToolInfo mediaToolInfo) return false; } - public bool InvokeLogFailedResult(BufferedCommandResult result) => LogFailedResult(result); + public bool InvokeLogFailedResult( + BufferedCommandResult result, + string fileName, + string operation + ) => LogFailedResult(result, fileName, operation); } // Call LogFailedResult with the given exit code and stderr, capturing the emitted event by // temporarily redirecting the static Serilog logger, then render it through the {Message} template // the console and file sinks use - private static string RenderLogFailedResult(int exitCode, string stderr) + private static string RenderLogFailedResult( + int exitCode, + string stderr, + string fileName = "file.mkv", + string stdout = "", + string operation = "Verify" + ) { CapturingSink sink = new(); ILogger original = Log.Logger; @@ -66,10 +76,10 @@ private static string RenderLogFailedResult(int exitCode, string stderr) exitCode, DateTimeOffset.MinValue, DateTimeOffset.MinValue, - string.Empty, + stdout, stderr ); - _ = tool.InvokeLogFailedResult(result).Should().BeFalse(); + _ = tool.InvokeLogFailedResult(result, fileName, operation).Should().BeFalse(); } finally { @@ -97,14 +107,29 @@ public void LogFailedResult_WithStderr_SeparatorStaysOutsideQuotes() // The error text is quoted as its own value, opening after the " : " separator _ = rendered.Should().Contain("ExitCode: 0 : \""); _ = rendered.Should().Contain(Stderr); + // The file name follows as the last quoted value + _ = rendered.Should().EndWith("\"file.mkv\""); // The stray-quote bug rendered "ExitCode: 0\" : ..." with the quote right after the number _ = rendered.Should().NotContain("ExitCode: 0\""); } [Fact] - public void LogFailedResult_WithoutStderr_HasNoTrailingSeparatorOrQuote() + public void LogFailedResult_WithoutStderr_HasNoErrorValueButKeepsFileName() { + // With no output on either stream the error value is omitted, leaving the operation, exit code + // and file name string rendered = RenderLogFailedResult(2, string.Empty); - _ = rendered.Should().Be("Failed execution of FfMpeg : ExitCode: 2"); + _ = rendered + .Should() + .Be("Failed execution of FfMpeg : Verify : ExitCode: 2 : \"file.mkv\""); + } + + [Fact] + public void LogFailedResult_ErrorOnStdout_FallsBackToStdout() + { + // A tool that writes its error to stdout with an empty stderr (e.g. mkvtoolnix) is still logged + const string StdoutError = "Error: the file could not be opened for reading"; + string rendered = RenderLogFailedResult(2, string.Empty, stdout: StdoutError); + _ = rendered.Should().Contain(StdoutError); } } From 2fad5371e17329c1fe65baac94560d670a7c3264 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 16 Jul 2026 05:45:12 +0000 Subject: [PATCH 14/19] Bump actions/setup-dotnet from 5.4.0 to 6.0.0 in the actions-deps group (#849) Bumps the actions-deps group with 1 update: [actions/setup-dotnet](https://github.com/actions/setup-dotnet). Updates `actions/setup-dotnet` from 5.4.0 to 6.0.0 - [Release notes](https://github.com/actions/setup-dotnet/releases) - [Commits](https://github.com/actions/setup-dotnet/compare/26b0ec14cb23fa6904739307f278c14f94c95bf1...a98b56852c35b8e3190ac28c8c2271da59106c68) --- updated-dependencies: - dependency-name: actions/setup-dotnet dependency-version: 6.0.0 dependency-type: direct:production update-type: version-update:semver-major dependency-group: actions-deps ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/build-executable-task.yml | 2 +- .github/workflows/get-version-task.yml | 2 +- .github/workflows/validate-task.yml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-executable-task.yml b/.github/workflows/build-executable-task.yml index c5d51a9c..562f3812 100644 --- a/.github/workflows/build-executable-task.yml +++ b/.github/workflows/build-executable-task.yml @@ -46,7 +46,7 @@ jobs: steps: - name: Setup .NET SDK step - uses: actions/setup-dotnet@26b0ec14cb23fa6904739307f278c14f94c95bf1 # v5.4.0 + uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 with: dotnet-version: 10.x diff --git a/.github/workflows/get-version-task.yml b/.github/workflows/get-version-task.yml index 6c22652b..3ab8f141 100644 --- a/.github/workflows/get-version-task.yml +++ b/.github/workflows/get-version-task.yml @@ -39,7 +39,7 @@ jobs: steps: - name: Setup .NET SDK step - uses: actions/setup-dotnet@26b0ec14cb23fa6904739307f278c14f94c95bf1 # v5.4.0 + uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 with: dotnet-version: 10.x diff --git a/.github/workflows/validate-task.yml b/.github/workflows/validate-task.yml index f3606ff7..a8dbd40d 100644 --- a/.github/workflows/validate-task.yml +++ b/.github/workflows/validate-task.yml @@ -21,7 +21,7 @@ jobs: steps: - name: Setup .NET SDK step - uses: actions/setup-dotnet@26b0ec14cb23fa6904739307f278c14f94c95bf1 # v5.4.0 + uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 with: dotnet-version: 10.x @@ -52,7 +52,7 @@ jobs: steps: - name: Setup .NET SDK step - uses: actions/setup-dotnet@26b0ec14cb23fa6904739307f278c14f94c95bf1 # v5.4.0 + uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 with: dotnet-version: 10.x From 107fc1a03e07b6c6698c04fdab662e271cb353df Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 16 Jul 2026 05:45:56 +0000 Subject: [PATCH 15/19] Bump the nuget-deps group with 2 updates (#851) Bumps ptr727.LanguageTags from 1.5.44 to 1.5.48 Bumps ptr727.Utilities from 4.0.15 to 4.0.18 --- updated-dependencies: - dependency-name: ptr727.LanguageTags dependency-version: 1.5.48 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: nuget-deps - dependency-name: ptr727.Utilities dependency-version: 4.0.18 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: nuget-deps ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Directory.Packages.props | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 4386353c..4a6153cd 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -6,8 +6,8 @@ - - + + From f15c6fbb697df3db3cd1bbb6011c3ef2980dc6b1 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Thu, 16 Jul 2026 10:36:52 -0700 Subject: [PATCH 16/19] Refresh repo-config carry to current reference; add self-audit carry (#854) Rename repo-config/ruleset-develop.json and ruleset-main.json to develop.json and main.json to match the current fleet reference layout (the fleet audit letter-checks the new paths), and refresh README.md, configure.sh, and settings.json to the current reference content. Add the adapted self-audit carry per the repo-config Downstream Carry: AUDIT.md (adapted from the blessed Vantage-Config reference for this repo's release model, docker-hub and codecov mechanisms) and spec/secrets.json. Sync .github/workflows/merge-bot-pull-request.yml to the current fleet reference, notably removing --delete-branch from gh pr merge and updating the header and comments. Live rulesets, settings, and secrets were verified in sync with the reference payloads today; this change touches committed files only. Co-authored-by: Claude Opus 4.8 (1M context) --- .github/workflows/merge-bot-pull-request.yml | 27 +- AUDIT.md | 73 +++++ repo-config/README.md | 123 ++++---- repo-config/configure.sh | 286 +++++++------------ repo-config/develop.json | 68 +++++ repo-config/main.json | 65 +++++ repo-config/ruleset-develop.json | 45 --- repo-config/ruleset-main.json | 44 --- repo-config/settings.json | 5 +- spec/secrets.json | 34 +++ 10 files changed, 417 insertions(+), 353 deletions(-) create mode 100644 AUDIT.md create mode 100644 repo-config/develop.json create mode 100644 repo-config/main.json delete mode 100644 repo-config/ruleset-develop.json delete mode 100644 repo-config/ruleset-main.json create mode 100644 spec/secrets.json diff --git a/.github/workflows/merge-bot-pull-request.yml b/.github/workflows/merge-bot-pull-request.yml index b500ed0e..69eb76c2 100644 --- a/.github/workflows/merge-bot-pull-request.yml +++ b/.github/workflows/merge-bot-pull-request.yml @@ -1,19 +1,18 @@ name: Merge bot pull request action -# Enable auto-merge once per PR on opened/reopened; disable it when a maintainer pushes to a bot branch. Merge -# method by base branch (develop = squash, main = merge). App token so the merge fires downstream workflows -# (GITHUB_TOKEN pushes don't) and so the disable job has write access on read-only Dependabot PRs. - -# `pull_request_target` (not `pull_request`): these jobs hold the App private key, so the workflow definition and -# its action SHAs must resolve from the trusted base branch, not the PR head. Safe because no job checks out PR -# code - each only runs `gh pr merge` against the PR by URL. +# Auto-merges in-repo Dependabot PRs: enable on opened/reopened, disable on a maintainer push. Carried from the +# fleet reference trimmed to the Dependabot jobs - no codegen bot opens PRs against this repo. +# - Merge method by base: develop = squash, main = merge. +# - App token, not GITHUB_TOKEN: fires downstream workflows on merge, and grants write on read-only Dependabot PRs. +# - pull_request_target, not pull_request: jobs hold the App key, so the workflow + action SHAs resolve from the +# trusted base, not PR head. Safe because no job checks out PR code (each runs gh pr merge by URL). on: pull_request_target: types: [opened, reopened, synchronize] -# Per-PR group: under `pull_request_target` `github.ref` is the base branch, which would serialize every bot PR -# against that base; key on the PR number so each PR's events queue independently. `cancel-in-progress: false` so a -# follow-up synchronize doesn't cancel an in-flight `opened` run before it enables auto-merge. +# Concurrency keys on the PR number, not github.ref (the base branch under pull_request_target, which would +# serialize every bot PR against it), so each PR queues independently. cancel-in-progress: false so a follow-up +# synchronize doesn't cancel an in-flight opened run before it enables auto-merge. concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number }} cancel-in-progress: false @@ -23,8 +22,7 @@ jobs: merge-dependabot: name: Merge dependabot pull request job runs-on: ubuntu-latest - # In-repo Dependabot PRs only, on opened/reopened so the disable job stays sticky. Every tier - # auto-merges, semver-major included: the required checks are the gate, not the version bump. + # Dependabot PRs from this repo (not forks). Only on opened/reopened so the disable job stays sticky. if: >- (github.event.action == 'opened' || github.event.action == 'reopened') && github.event.pull_request.user.login == 'dependabot[bot]' && @@ -42,6 +40,7 @@ jobs: client-id: ${{ secrets.CODEGEN_APP_CLIENT_ID }} private-key: ${{ secrets.CODEGEN_APP_PRIVATE_KEY }} + # Auto-merge every tier, semver-major included: the required checks are the gate, not the bump magnitude. - name: Merge pull request step run: | set -euo pipefail @@ -53,7 +52,7 @@ jobs: exit 1 ;; esac - gh pr merge --auto --delete-branch "$method" "$PR_URL" + gh pr merge --auto "$method" "$PR_URL" env: PR_URL: ${{ github.event.pull_request.html_url }} GH_TOKEN: ${{ steps.app-token.outputs.token }} @@ -62,7 +61,7 @@ jobs: name: Disable auto-merge on maintainer push job runs-on: ubuntu-latest # Fires when a maintainer pushes to a bot's branch (synchronize, actor != bot). Disables auto-merge so the - # maintainer's commits don't merge with the bot's; they re-enable it manually. The disable call is idempotent. + # maintainer's commits don't merge with the bot's, and they re-enable it manually. The disable call is idempotent. if: >- github.event.action == 'synchronize' && github.event.pull_request.head.repo.full_name == github.repository && diff --git a/AUDIT.md b/AUDIT.md new file mode 100644 index 00000000..7913cb90 --- /dev/null +++ b/AUDIT.md @@ -0,0 +1,73 @@ +# AUDIT.md + +How this repository audits itself against its committed baseline and reports drift. This is the repo-scoped adaptation of the fleet-wide AUDIT.md kept at the fleet hub (carried per the [repo-config downstream carry][repo-config-readme]); the hub's fleet-wide audit remains authoritative. The ground truth here is the committed [`repo-config/`][repo-config] payloads and [`spec/secrets.json`][secrets]; the prose authorities are [`AGENTS.md`][agents], [`CODESTYLE.md`][codestyle], and [`WORKFLOW.md`][workflow]. + +The audit is read-only: it diffs live state against the committed baseline and reports findings; it never applies changes. The verdict vocabulary is [`WORKFLOW.md`][workflow]'s: **operational / not operational**, **N/A**, **defect**, and the applicable/absent rule. + +## Scope + +This is a release-model repo: the self-audit covers the `main` and `develop` rulesets, general repository settings, and secret names. Code-project conformance (analyzers, tests, coverage, publish workflows) is CI's job and the fleet hub's fleet-wide audit's, not this self-audit's - see [AGENTS.md "Branching Model"][agents-branching-model] for the model this baseline encodes. + +## General Settings + +Diff the live repository settings against [`repo-config/settings.json`][repo-config-settings]. The two state-dependent settings are not in the file: `has_discussions` follows visibility (public on / private off) and `default_branch` is `main`. + +```sh +repo="$(gh repo view --json nameWithOwner --jq '.nameWithOwner')" +live=$(gh api "repos/$repo" --jq '{has_wiki,has_projects,allow_merge_commit,allow_squash_merge,allow_rebase_merge,allow_auto_merge,allow_update_branch,delete_branch_on_merge}') +diff <(jq -S . repo-config/settings.json) <(jq -S . <<<"$live") \ + && echo "settings: in sync" || echo "settings: DRIFT" +``` + +## Rulesets + +Diff each live ruleset against the committed expected payload with a normalized comparison (sort the order-insensitive `rules[]` and `bypass_actors[]` before diffing so a reordered but equivalent ruleset does not read as drift). This release carry keeps its `develop` payload at [`repo-config/develop.json`][repo-config-develop]. + +```sh +repo="$(gh repo view --json nameWithOwner --jq '.nameWithOwner')" +norm='{name,target,enforcement,bypass_actors,conditions,rules} | .rules|=sort_by(.type) | .bypass_actors|=sort_by(.actor_id)' +for b in develop main; do + file="repo-config/$b.json" + id=$(gh api "repos/$repo/rulesets" --jq ".[]|select(.name==\"$b\").id") + diff <(jq -S "$norm" "$file") \ + <(gh api "repos/$repo/rulesets/$id" --jq '{name,target,enforcement,bypass_actors,conditions,rules}' | jq -S "$norm") \ + && echo "$b: in sync" || echo "$b: DRIFT" +done +``` + +The result must be exactly two rulesets named `develop` and `main` - a missing ruleset or a divergent payload is a **defect**; a duplicate or stray ruleset is a **drift finding**. + +## Secrets + +Confirm each name [`spec/secrets.json`][secrets] requires exists in the stores its mechanism claims, and no forbidden name is present (names only; values are not readable). The baseline App pair and the Docker Hub pair live in both the Actions and Dependabot stores; `CODECOV_TOKEN` is claimed in the Actions store. + +```sh +repo="$(gh repo view --json nameWithOwner --jq '.nameWithOwner')" +for store in actions dependabot; do + names=$(gh api "repos/$repo/$store/secrets" --jq '.secrets[].name') + want="CODEGEN_APP_CLIENT_ID CODEGEN_APP_PRIVATE_KEY DOCKER_HUB_USERNAME DOCKER_HUB_ACCESS_TOKEN" + [ "$store" = "actions" ] && want="$want CODECOV_TOKEN" + for s in $want; do + grep -qx "$s" <<<"$names" && echo "$store/$s: present" || echo "$store/$s: MISSING (defect)" + done + for s in CODEGEN_APP_ID; do + grep -qx "$s" <<<"$names" && echo "$store/$s: forbidden name present (defect)" || true + done +done +``` + +## Verdict and Follow-Up + +A missing required item or a divergent payload is a **defect** (not operational); an equivalent outcome in a non-standard form is a **drift finding**. N/A items are excluded, never counted as failures. Surface findings as repository issues; fixes land as a pull request to `develop` per [AGENTS.md "Branching Model"][agents-branching-model]. To re-apply the whole baseline, run `repo-config/configure.sh` (see [repo-config/README.md][repo-config-readme]). + + + +[agents]: ./AGENTS.md +[agents-branching-model]: ./AGENTS.md#branching-model +[codestyle]: ./CODESTYLE.md +[repo-config]: ./repo-config/ +[repo-config-develop]: ./repo-config/develop.json +[repo-config-readme]: ./repo-config/README.md +[repo-config-settings]: ./repo-config/settings.json +[secrets]: ./spec/secrets.json +[workflow]: ./WORKFLOW.md diff --git a/repo-config/README.md b/repo-config/README.md index 08afdd55..658fdd07 100644 --- a/repo-config/README.md +++ b/repo-config/README.md @@ -1,68 +1,71 @@ # repo-config -Repository configuration as code - the parts of "operational" that live in GitHub settings rather than -in workflow YAML: branch rulesets, repository settings, and the secrets the workflows read. This is the -concrete form of [`WORKFLOW.md`](../WORKFLOW.md) section 6 and guarantee **D10**, and the implementation -of its **5D configuration audit**. - -This directory is intentionally **not** under `.github/` - that path is GitHub's own (workflows, issue -templates); repository administration config-as-code is the maintainer's, so it lives here. - -## Files - -- [`configure.sh`](./configure.sh) - idempotent `gh api` script with two modes: - - `./repo-config/configure.sh check` - validate only, no writes; exits non-zero on drift (the 5D - audit). Read-only, but it reads the rulesets and secrets endpoints, so it still needs a `gh` token - with admin on the repo. - - `./repo-config/configure.sh apply` - create-or-update the rulesets and settings to match this - directory (needs admin; writes). -- [`ruleset-develop.json`](./ruleset-develop.json) - the `develop` branch ruleset (squash-only, linear - history, signed commits, the required status check, strict-status **off**). -- [`ruleset-main.json`](./ruleset-main.json) - the `main` branch ruleset (merge-commit-only, signed - commits, the same required check, strict **off**; no linear-history rule). -- [`settings.json`](./settings.json) - repository settings (auto-merge on; squash **and** merge-commit - allowed; rebase off; auto-delete-on-merge **off**). The repo-wide auto-delete **setting** is off so a - `develop -> main` promotion does not delete `develop` (GitHub's auto-delete would remove the merged head - branch). Per-merge deletion is explicit instead: the merge-bot deletes a merged bot branch with - `gh pr merge --delete-branch`, and a feature branch is deleted the same way (or via the merge UI's delete - button) - so `main`/`develop` survive while bot/feature branches are still cleaned up. - -## What it does not store - -Secret **values** are never readable through the API, so the script only asserts the required secret -**names** exist (`DOCKER_HUB_USERNAME` / `DOCKER_HUB_ACCESS_TOKEN` for the image, the App credentials -`CODEGEN_APP_CLIENT_ID` / `CODEGEN_APP_PRIVATE_KEY` for the merge-bot, and `CODECOV_TOKEN` for the -report-only coverage upload), and *notes* (best-effort) whether a -GitHub App is installed - a precise check needs app-level auth, so the App-installation check does not fail the audit. -The Docker Hub, App, and Codecov credentials must be set in **both** the Actions and Dependabot secret stores, since a -Dependabot-triggered run gets the Dependabot store. Set the values in the repository (or organization) secret store directly. There is no -NuGet publishing here; the GitHub release uses the built-in `GITHUB_TOKEN`. The Docker Hub access token's -validity and push scope are verified by hand, not by this script. - -## Applying, and the required-check rename lockstep - -The live ruleset's required status check is matched by **name** to the aggregator job in -[`test-pull-request.yml`](../.github/workflows/test-pull-request.yml) (`Check pull request workflow -status job`). GitHub binds the check by that exact string, so the ruleset JSON here, the live ruleset, and -the aggregator job name must move **in lockstep** ([`WORKFLOW.md`](../WORKFLOW.md) D6.2). If they drift, a -pull request runs CI but its required check never resolves and the PR cannot merge. - -So whenever the ruleset JSON or that job name changes, run `apply` against the live repo in the same -change that ships the workflow edit, then `check`: +Repository and branch configuration held as committed files, kept out of `.github/` (which holds the GitHub-consumed configuration - workflows, Dependabot). This mirrors the layout the fleet repos use. + +- `main.json` plus one `develop` variant - the branch rulesets as the writable API subset (`name`, `target`, `enforcement`, `bypass_actors`, `conditions`, `rules`). The `develop` payload is `develop.json` (`release` repos) or `operational/develop.json` (`operational` repos); the hub keeps both, a carried copy only its own model's (see "Downstream Carry"). These are the canonical expected payloads that the audit (the hub's fleet-wide `AUDIT.md`, or a carried repo-scoped adaptation - see "Downstream Carry") diffs the live rulesets against. +- `operational/develop.json` - the `develop` ruleset for **operational** repos (registry `workflowModel: operational`): direct signed pushes, no PR gate. Present at the hub and in operational carries only - a carried `release` repo does not have it. See "Rulesets" below. +- `configure.sh` - applies the rulesets to a repository via the GitHub API (create or full-payload update, idempotent). Run `repo-config/configure.sh [owner/repo] [release|operational]`; the model defaults to the registry `workflowModel` lookup. + +## Downstream Carry + +Every fleet repo carries this directory; the hub keeps the canonical copy. Rules for the carried copy: + +- **Carry only your model's `develop` variant.** A `release` repo carries `develop.json`; an `operational` repo carries `operational/develop.json` instead. `main.json` and `settings.json` are shared by both models. `configure.sh` aborts when the payload its model needs is missing rather than applying a partial configuration. +- **Hub-only references stay plain text.** The hub is a private repo: never URL-link it from a downstream repo - the link 404s for anyone without hub access. Files whose canonical fleet-wide form lives only at the hub are mentioned by name, not linked; links into files every repo carries (`AGENTS.md`) resolve everywhere and are fine. +- **Adapted self-audit carry.** A downstream repo carries **locally adapted** `AUDIT.md` and `spec/secrets.json`, scoped to self-auditing its own rulesets, settings, and secrets against the committed `repo-config/` baseline - the standard shape, so the carried tooling is self-contained. The hub's fleet-wide audit remains authoritative, and the local copies never link the hub. The reference adaptation is the [Vantage-Config carry][vantage-config] (`AUDIT.md` + `spec/secrets.json`, operational model): a settings diff, a normalized ruleset diff against the carried payloads, and a names-only secrets check, all targeting the current repo - adapt it, don't invent. A `release` repo adapts the same shape: its `develop` payload stays at `repo-config/develop.json`, and its `spec/secrets.json` keeps the baseline App pair plus the secret names for its own publish mechanisms (from the hub's canonical `spec/secrets.json`). The fleet audit letter-checks both carried files (`spec/files.json`). +- **The regen snippet targets the current repo**, so it works unchanged in a carried copy. + +## Rulesets + +Two workflow models share `main.json` but differ on `develop` (registry `workflowModel`, default `release`): + +- **`release`** (`develop.json`): `develop` requires squash merges with linear history and a PR - the feature-branch pipeline. +- **`operational`** (`operational/develop.json`): `develop` takes **direct signed pushes** - only `deletion`, `non_fast_forward`, and `required_signatures`; no PR, no status-check, no Copilot-on-push. CI runs on the push as advisory feedback. This is for live-service config repos that edit `develop` directly and promote a known-good snapshot to `main` via an occasional PR (see [AGENTS.md "Branching Model"][agents-branching-model]). + +`main` (both models) requires merge-commit merges (no linear-history rule), signed commits, a passing `Check pull request workflow status job`, resolved review threads, and Copilot review, and blocks force-pushes and deletion - so a `develop -> main` promotion is always gated even when `develop` takes direct commits. Every ruleset intentionally leaves "Require branches to be up to date before merging" **off** - see [AGENTS.md "Branching Model"][agents-branching-model]. + +**Configure by importing these JSON files, never by hand-building the rules** (hand reconstruction has gone wrong on past setups). The result must be **exactly two rulesets named `develop` and `main`** - the names are load-bearing (`AGENTS.md` and the workflows reference them); only the `develop` *content* varies by model. First remove all legacy classic branch-protection rules and any stray rulesets, then run `configure.sh` (which picks the `develop` payload from the repo's `workflowModel`), or `gh api -X POST repos///rulesets --input repo-config/.json` per file (operational repos use `operational/develop.json` for `develop`). `gh ruleset` is read-only; creation goes through `gh api`. The required check binds by name and only turns green after the repo's PR workflow runs once. To edit a ruleset, GET it, change the field, and PUT the whole writable subset back (a partial PUT `422`s). + +To change the canonical rulesets, edit the live rulesets (fleet-wide changes happen at the hub), then regenerate the committed files from the current repo: ```sh -REPO=ptr727/PlexCleaner ./repo-config/configure.sh apply # sync live rulesets + settings + security -REPO=ptr727/PlexCleaner ./repo-config/configure.sh check # confirm no drift +repo="$(gh repo view --json nameWithOwner --jq '.nameWithOwner')" +for name in develop main; do + out="repo-config/$name.json" + # An operational carry keeps its develop payload at operational/develop.json (develop.json is absent). + [ "$name" = "develop" ] && [ ! -e "$out" ] && out="repo-config/operational/develop.json" + id=$(gh api "repos/$repo/rulesets" --jq ".[] | select(.name==\"$name\") | .id") + gh api "repos/$repo/rulesets/$id" \ + --jq '{name, target, enforcement, bypass_actors, conditions, rules}' \ + | jq -S --indent 4 '.' > "$out" +done ``` -First-time adoption is the same step: the live ruleset predates the renamed aggregator, so the first -`apply` is what lets a pull request against the new workflows go green. Both modes need a `gh` login -with admin on the repo (the rulesets and secrets endpoints require it). `apply` writes, `check` only -reads. +## Secrets + +Publish credentials required per mechanism are enumerated in `spec/secrets.json` (canonical at the hub; a downstream repo carries a repo-scoped adaptation - see "Downstream Carry"). A repo needs only the mechanisms its own publish targets use - a source-only repo needs none of the publish credentials below. NuGet and PyPI use keyless OIDC Trusted Publishing (no stored key; the publish job needs `id-token: write`, and PyPI additionally an `environment: pypi` gate). Docker Hub has no OIDC equivalent and uses a stored `DOCKER_HUB_USERNAME` + `DOCKER_HUB_ACCESS_TOKEN` in both the Actions and Dependabot secret stores. Codegen and merge-bot repos add a GitHub App (`CODEGEN_APP_CLIENT_ID` + `CODEGEN_APP_PRIVATE_KEY` in both stores; the app must be installed, not just created). App-token call sites use `client-id`, never the deprecated `app-id`. + +## Repo Settings + +The fleet-standard general settings live in [`settings.json`][settings-json] and are applied idempotently by `configure.sh` alongside the rulesets (`gh api PATCH /repos/{owner}/{repo}`). The two settings that depend on per-repo state - `has_discussions` (visibility) and `default_branch` (main-must-exist) - are computed by the script, not stored in the file. + +- **Default branch `main`** (the script sets it only when a `main` branch exists, never pointing the default at a missing branch). +- **Merge methods**: `Allow merge commits` and `Allow squash merging` on, **rebase off** - each branch ruleset then picks its method (merge on `main`, squash on `develop`). +- **Auto-merge on** (the merge-bot needs it) and **`Always suggest updating pull request branches` on**. +- **`Automatically delete head branches` OFF - deliberately.** With it on, a `develop -> main` promotion (whose PR head is `develop`) would delete `develop`. There is no per-branch exemption, so the repo-wide toggle stays off to protect `develop`. **The CLI has the same trap: never `gh pr merge --delete-branch` a promotion PR whose head is `develop`** - the explicit flag deletes `develop` regardless of this setting (see [AGENTS.md "Branching Model"][agents-branching-model]). +- **Wikis and Projects off. Discussions on public repos only** (off on private). **Sponsorships off** - the button is driven by `.github/FUNDING.yml`, not a REST toggle, and the fleet ships none. +- **Actions / General**: allow GitHub Actions to create and approve pull requests (for the bots). + +## Brownfield Migration (Maintainer Only) + +`Require signed commits` rejects any pre-existing unsigned commit, so the first `develop -> main` release on a repo with unsigned history is blocked. Re-signing that history is a non-fast-forward that the `Block force pushes` rule rejects, **and the admin bypass does not cover `git push --force`**. Completing it requires temporarily disabling the ruleset and a maintainer force-push. This is a one-time, maintainer-performed migration that deliberately uses the force-push [AGENTS.md "Git and Commit Rules"][agents-git-and-commit-rules] forbids agents from running - **an agent must never execute it; surface it to the maintainer**. Greenfield repos where signing is live before the first commit never hit this. + + + +[agents-branching-model]: ../AGENTS.md#branching-model +[agents-git-and-commit-rules]: ../AGENTS.md#git-and-commit-rules +[settings-json]: ./settings.json -## Why both a script and JSON + -The JSON files are the unambiguous source of truth for the configuration; the script applies and audits -them idempotently. An agent can also derive the same checks on the fly from `WORKFLOW.md` section 6, but -the committed script and JSON codify the exact intended state so the configuration is reproducible and -diffable rather than tribal knowledge. +[vantage-config]: https://github.com/ptr727/Vantage-Config diff --git a/repo-config/configure.sh b/repo-config/configure.sh index 9363481e..6e2b228c 100755 --- a/repo-config/configure.sh +++ b/repo-config/configure.sh @@ -1,194 +1,102 @@ #!/usr/bin/env bash -# Repository configuration as code - the secrets, branch rulesets, and settings the workflows assume -# (see WORKFLOW.md section 6, guarantee D10). Idempotent: `apply` configures a repo to match the JSON in -# this directory; `check` validates an existing repo and exits non-zero on drift (the 5D audit). Run from -# anywhere; the target repo is resolved from the current `gh` context unless $REPO is set (owner/name). +# Apply the committed fleet configuration in this directory to the repository via the GitHub API: +# 1. General repository settings from settings.json (PATCH /repos/{owner}/{repo}), plus the two settings that depend on +# per-repo state - has_discussions (public repos only) and default_branch (main, only if it exists). +# 2. The branch rulesets. main.json is shared by both workflow models; the develop ruleset is model-specific - +# release repos use develop.json (PR-gated), operational repos use operational/develop.json (direct signed +# pushes). The model is read from ../registry/repos.json (per-repo workflowModel, else defaults.workflowModel, +# else release) and can be overridden with the second argument. Each .json holds the writable ruleset +# subset {name, target, enforcement, bypass_actors, conditions, rules}. An existing ruleset (matched by name) +# is updated with a full-payload PUT (partial PUTs 422); a missing one is created with POST. +# Rerunning is idempotent. # -# ./repo-config/configure.sh check # validate only, no writes (the 5D audit) -# ./repo-config/configure.sh apply # create-or-update rulesets + settings (writes) -# -# Requires gh and jq. Both modes read the rulesets and secrets endpoints, which need admin on the repo, so -# gh must be authenticated with admin for `check` as well as `apply`. `check` only reads; `apply` writes. - +# Usage: repo-config/configure.sh [owner/repo] [release|operational] (repo defaults to the current repo via gh; +# model defaults to the registry lookup) set -euo pipefail -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO="${REPO:-$(gh repo view --json nameWithOwner --jq .nameWithOwner)}" - -# Secrets by store (names only; values are never readable via the API). The Docker Hub credentials, the -# merge-bot App credentials, and CODECOV_TOKEN must be set in BOTH stores: a Dependabot-triggered run gets the -# Dependabot secret store, not Actions secrets, and that run's push CI builds the Docker smoke (logs in to -# Docker Hub) and runs the validate job (uploads coverage to Codecov). Publishing the GitHub release uses the -# built-in GITHUB_TOKEN (no secret needed). -REQUIRED_ACTIONS_SECRETS=(DOCKER_HUB_USERNAME DOCKER_HUB_ACCESS_TOKEN CODEGEN_APP_CLIENT_ID CODEGEN_APP_PRIVATE_KEY CODECOV_TOKEN) -REQUIRED_DEPENDABOT_SECRETS=(DOCKER_HUB_USERNAME DOCKER_HUB_ACCESS_TOKEN CODEGEN_APP_CLIENT_ID CODEGEN_APP_PRIVATE_KEY CODECOV_TOKEN) -REQUIRED_CHECK="Check pull request workflow status job" - -note() { printf ' %s\n' "$*"; } -pass() { printf ' \033[32mok\033[0m %s\n' "$*"; } -fail() { printf ' \033[31mFAIL\033[0m %s\n' "$*"; FAILED=1; } -FAILED=0 - -ruleset_id() { # name -> id (empty if absent); aborts with a visible reason on an API error - local out - # An absent ruleset is a successful call with no match (empty); only a real API error fails. Let gh print its - # own error on stderr (do not suppress it); add a generic context line and return non-zero so the run stops - # (the caller's $(...) cannot print the cause itself). - # per_page=100 returns every ruleset in one array (a repo has only a handful); the default page size is 30. - if ! out="$(gh api "repos/$REPO/rulesets?per_page=100")"; then - echo "ERROR: could not list rulesets for $REPO (see gh error above)" >&2 - return 1 - fi - # shellcheck disable=SC2016 # $n is a jq variable (--arg n), not a shell expansion - # Select the first match inside jq (not `| head -1`): under pipefail, head closing the pipe early can - # SIGPIPE jq and fail the function. - jq -r --arg n "$1" '[.[] | select(.name==$n) | .id] | first // empty' <<<"$out" -} - -apply_ruleset() { - local file="$1" name id - name="$(jq -r .name "$file")" - id="$(ruleset_id "$name")" - if [[ -n "$id" ]]; then - gh api -X PUT "repos/$REPO/rulesets/$id" --input "$file" >/dev/null - note "updated ruleset '$name' (#$id)" - else - gh api -X POST "repos/$REPO/rulesets" --input "$file" >/dev/null - note "created ruleset '$name'" - fi -} - -cmd_apply() { - echo "Applying repository configuration to $REPO" - apply_ruleset "$DIR/ruleset-develop.json" - apply_ruleset "$DIR/ruleset-main.json" - gh api -X PATCH "repos/$REPO" --input "$DIR/settings.json" >/dev/null - note "patched repository settings" - gh api -X PUT "repos/$REPO/vulnerability-alerts" >/dev/null - gh api -X PUT "repos/$REPO/automated-security-fixes" >/dev/null - note "enabled Dependabot alerts + security updates" - echo "Done. Run '$0 check' to validate." -} - -# --- validation (5D) ------------------------------------------------------------------------------- - -# assert MESSAGE TEST... - run the test command; pass on success, fail on non-zero (proper if/else, not -# the A && B || C footgun). The test command may read stdin (e.g. a `<<<` heredoc on the assert call). -# Do not redirect the assert call's stdout - that would also swallow the pass/fail line; commands that -# print (jq) use `jq_has`, which silences only itself. -assert() { - local msg="$1"; shift - if "$@"; then pass "$msg"; else fail "$msg"; fi -} - -# jq_has FILTER... - true iff the jq filter selects something; jq's own output is discarded, not the -# caller's. Reads JSON from stdin. -jq_has() { jq -e "$@" >/dev/null 2>&1; } - -# jq_lacks FILTER... - true iff the jq filter yields no truthy value (selects nothing, or only false/null). -# `jq -e` exits 1 (last output false/null) or 4 (no output at all) for the "lacks" cases, 0 for a truthy -# match, and 2/3/5 for a real error (malformed filter or input), which is propagated so the calling assert -# fails loudly. The `|| rc=$?` keeps jq in a list (exempt from set -e) so a non-zero exit captures rc instead -# of aborting. Only stdout is discarded - jq's stderr is kept so a real error shows its diagnostic. -jq_lacks() { local rc=0; jq -e "$@" >/dev/null || rc=$?; case "$rc" in 0) return 1 ;; 1|4) return 0 ;; *) return "$rc" ;; esac; } - -check_ruleset() { # name expected-merge-method expect-linear(true/false) - local name="$1" method="$2" linear="$3" id rs - id="$(ruleset_id "$name")" - if [[ -z "$id" ]]; then fail "ruleset '$name' missing"; return; fi - rs="$(gh api "repos/$REPO/rulesets/$id")" - assert "ruleset '$name' active" \ - test "$(jq -r '.enforcement' <<<"$rs")" = active - assert "'$name' merge method = $method" \ - test "$(jq -r '.rules[] | select(.type=="pull_request") | .parameters.allowed_merge_methods | join(",")' <<<"$rs")" = "$method" - assert "'$name' requires signed commits" \ - jq_has '.rules[] | select(.type=="required_signatures")' <<<"$rs" - assert "'$name' strict status policy off" \ - test "$(jq -r '.rules[] | select(.type=="required_status_checks") | .parameters.strict_required_status_checks_policy' <<<"$rs")" = false - # shellcheck disable=SC2016 # $c is a jq variable (--arg c), not a shell expansion - assert "'$name' requires '$REQUIRED_CHECK'" \ - jq_has --arg c "$REQUIRED_CHECK" '.rules[] | select(.type=="required_status_checks") | .parameters.required_status_checks[] | select(.context==$c)' <<<"$rs" - if [[ "$linear" == "true" ]]; then - assert "'$name' requires linear history" \ - jq_has '.rules[] | select(.type=="required_linear_history")' <<<"$rs" - else - # main must NOT require linear history - it would block the develop -> main merge-commit promotion. - assert "'$name' does not require linear history" \ - jq_lacks '.rules[] | select(.type=="required_linear_history")' <<<"$rs" - fi -} - -# gh_ok ENDPOINT... - true iff the gh api call succeeds (2xx, including 204). Output and errors are -# discarded, so it is safe to pass to `assert`. -gh_ok() { gh api "$@" >/dev/null 2>&1; } - -check_settings() { - local s; s="$(gh api "repos/$REPO")" - # Drive every assertion from settings.json, so the check covers exactly the applied desired state and - # never drifts from the file (add a key there and it is audited here automatically). - local key want got - while IFS=$'\t' read -r key want; do - # shellcheck disable=SC2016 # $k is a jq variable (--arg k), not a shell expansion - got="$(jq -r --arg k "$key" '.[$k]' <<<"$s")" - assert "setting $key = $want" test "$got" = "$want" - done < <(jq -r 'to_entries[] | "\(.key)\t\(.value)"' "$DIR/settings.json") -} - -check_security() { - # apply enables both; audit that they are still on. vulnerability-alerts returns 204 when enabled and - # 404 when disabled; automated-security-fixes returns { "enabled": true/false }. - assert "Dependabot vulnerability alerts enabled" gh_ok "repos/$REPO/vulnerability-alerts" - assert "Dependabot automated security updates enabled" \ - jq_has '.enabled == true' < <(gh api "repos/$REPO/automated-security-fixes") -} - -check_secrets() { - # --paginate: the secrets endpoints page at 30, so without it a repo with many secrets could miss a - # required name and report a false failure. An API/auth error FAILs fast (the required secrets cannot be - # verified, so reporting "matches" would be wrong) - distinct from a genuinely missing secret, which also - # FAILs. gh prints its own error (stderr not suppressed) so the cause is actionable. - local actions deps - if ! actions="$(gh api --paginate "repos/$REPO/actions/secrets" --jq '.secrets[].name')"; then - fail "could not list Actions secrets (API error - cannot verify required secrets)"; return - fi - if ! deps="$(gh api --paginate "repos/$REPO/dependabot/secrets" --jq '.secrets[].name')"; then - fail "could not list Dependabot secrets (API error - cannot verify required secrets)"; return - fi - for s in "${REQUIRED_ACTIONS_SECRETS[@]}"; do - assert "actions secret $s present" grep -qx "$s" <<<"$actions" - done - for s in "${REQUIRED_DEPENDABOT_SECRETS[@]}"; do - assert "dependabot secret $s present" grep -qx "$s" <<<"$deps" - done -} - -check_app() { - # Best-effort: confirm a GitHub App installation backs the merge-bot automation. A precise check - # requires app-level auth; presence of the App secrets above is the practical proxy. - if gh api "repos/$REPO/installation" >/dev/null 2>&1; then - pass "a GitHub App is installed on the repo" - else - note "could not confirm App installation via this token (verify the merge-bot App is installed)" - fi -} - -cmd_check() { - echo "Validating repository configuration for $REPO" - check_ruleset develop squash true - check_ruleset main merge false - check_settings - check_security - check_secrets - check_app - # Not checkable via gh api beyond name presence: that DOCKER_HUB_ACCESS_TOKEN is valid and has push - # access to docker.io/ptr727/plexcleaner. Verify it by hand in the Docker Hub account. - note "verify manually: Docker Hub access token valid with push to docker.io/ptr727/plexcleaner" - if [[ "$FAILED" -ne 0 ]]; then echo "Configuration drift detected."; exit 1; fi - echo "Configuration matches." -} - -case "${1:-check}" in - apply) cmd_apply ;; - check) cmd_check ;; - *) echo "usage: $0 [apply|check]" >&2; exit 2 ;; +repo="${1:-$(gh repo view --json nameWithOwner --jq '.nameWithOwner')}" +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# ----- Resolve the workflow model (selects the develop ruleset) ----- +registry="$script_dir/../registry/repos.json" +name="${repo##*/}" +model="${2:-}" +if [ -z "$model" ]; then + if [ -f "$registry" ]; then + # Fail fast on a jq/parse error (malformed registry) instead of silently applying the release default + # to a repo whose lookup actually broke. A repo simply absent from the registry is not an error: the + # expression falls back through defaults.workflowModel to "release", so jq still exits 0 with a value. + if ! model="$(jq -r --arg n "$name" '(.repos[] | select(.name==$n) | .workflowModel) // .defaults.workflowModel // "release"' "$registry")"; then + echo "Failed to read workflowModel from $registry (invalid JSON?). Pass the model explicitly as arg 2." >&2 + exit 1 + fi + else + # No registry to consult (e.g. running the script standalone) - default, but say so. + echo "Registry $registry not found; defaulting workflow model to release." >&2 + model="release" + fi +fi +case "$model" in + release) develop_ruleset="$script_dir/develop.json" ;; + operational) develop_ruleset="$script_dir/operational/develop.json" ;; + *) echo "Unknown workflow model '$model' (expected release or operational)." >&2; exit 1 ;; esac +echo "Workflow model for $repo: $model" + +# ----- General repository settings ----- +settings_file="$script_dir/settings.json" +if [ -e "$settings_file" ]; then + # has_discussions: enabled on public repos only (fleet policy); never on private. + private="$(gh api "repos/$repo" --jq '.private')" + disc=false; [ "$private" = "false" ] && disc=true + # default_branch main, but only point it at main when main exists - never set the default to a missing + # branch (e.g. a repo still on a rework branch). + if gh api "repos/$repo/branches/main" --jq '.name' >/dev/null 2>&1; then + payload="$(jq --argjson d "$disc" '. + {has_discussions: $d, default_branch: "main"}' "$settings_file")" + else + payload="$(jq --argjson d "$disc" '. + {has_discussions: $d}' "$settings_file")" + echo "Warning: $repo has no 'main' branch; leaving default_branch unchanged." >&2 + fi + echo "Applying general settings to $repo (has_discussions=$disc)" + printf '%s' "$payload" | gh api --method PATCH "repos/$repo" --input - >/dev/null +fi + +# ----- Branch rulesets ----- +# main.json is shared; the develop ruleset was selected by workflow model above. A missing or nameless +# payload aborts - silently skipping it would report success on a partially-applied configuration. +for file in "$develop_ruleset" "$script_dir/main.json"; do + if [ ! -e "$file" ]; then + echo "Ruleset payload $file not found; aborting to avoid a partially-applied configuration." >&2 + exit 1 + fi + ruleset_name="$(jq -r '.name // empty' "$file")" + if [ -z "$ruleset_name" ]; then + echo "Ruleset payload $file has no name; aborting to avoid a partially-applied configuration." >&2 + exit 1 + fi + # Paginate so a name match on a later page is never missed (which would create a duplicate ruleset), and + # fail loudly if the API call itself fails (auth/404/network) rather than treating it as "not found". + if ! ids="$(gh api --paginate "repos/$repo/rulesets" --jq ".[] | select(.name==\"$ruleset_name\") | .id")"; then + echo "Failed to list rulesets for $repo (check auth and repo access)." >&2 + exit 1 + fi + # Pre-existing drift can leave more than one ruleset with the same name; update the first and warn. Guard + # on non-empty so `grep -c` (which exits non-zero on empty input under `set -e`) can't abort the create path. + id="" + if [ -n "$ids" ]; then + count="$(printf '%s\n' "$ids" | grep -c .)" + if [ "$count" -gt 1 ]; then + echo "Warning: $count rulesets named '$ruleset_name' on $repo; updating the first (resolve the duplicates)." >&2 + fi + id="$(printf '%s\n' "$ids" | sed -n '1p')" + fi + if [ -n "$id" ]; then + echo "Updating ruleset '$ruleset_name' (id $id) on $repo" + gh api --method PUT "repos/$repo/rulesets/$id" --input "$file" >/dev/null + else + echo "Creating ruleset '$ruleset_name' on $repo" + gh api --method POST "repos/$repo/rulesets" --input "$file" >/dev/null + fi +done + +echo "Configuration applied to $repo" diff --git a/repo-config/develop.json b/repo-config/develop.json new file mode 100644 index 00000000..efc4262b --- /dev/null +++ b/repo-config/develop.json @@ -0,0 +1,68 @@ +{ + "bypass_actors": [ + { + "actor_id": 5, + "actor_type": "RepositoryRole", + "bypass_mode": "always" + } + ], + "conditions": { + "ref_name": { + "exclude": [], + "include": [ + "refs/heads/develop" + ] + } + }, + "enforcement": "active", + "name": "develop", + "rules": [ + { + "type": "deletion" + }, + { + "type": "non_fast_forward" + }, + { + "type": "required_linear_history" + }, + { + "type": "required_signatures" + }, + { + "parameters": { + "allowed_merge_methods": [ + "squash" + ], + "dismiss_stale_reviews_on_push": true, + "require_code_owner_review": false, + "require_last_push_approval": false, + "required_approving_review_count": 0, + "required_review_thread_resolution": true, + "required_reviewers": [] + }, + "type": "pull_request" + }, + { + "parameters": { + "do_not_enforce_on_create": false, + "required_status_checks": [ + { + "context": "Check pull request workflow status job", + "integration_id": 15368 + } + ], + "strict_required_status_checks_policy": false + }, + "type": "required_status_checks" + }, + { + "parameters": { + "review_draft_pull_requests": true, + "review_on_push": true + }, + "type": "copilot_code_review" + } + ], + "target": "branch" +} diff --git a/repo-config/main.json b/repo-config/main.json new file mode 100644 index 00000000..5d8d7b2d --- /dev/null +++ b/repo-config/main.json @@ -0,0 +1,65 @@ +{ + "bypass_actors": [ + { + "actor_id": 5, + "actor_type": "RepositoryRole", + "bypass_mode": "always" + } + ], + "conditions": { + "ref_name": { + "exclude": [], + "include": [ + "refs/heads/main" + ] + } + }, + "enforcement": "active", + "name": "main", + "rules": [ + { + "type": "deletion" + }, + { + "type": "non_fast_forward" + }, + { + "type": "required_signatures" + }, + { + "parameters": { + "allowed_merge_methods": [ + "merge" + ], + "dismiss_stale_reviews_on_push": true, + "require_code_owner_review": false, + "require_last_push_approval": false, + "required_approving_review_count": 0, + "required_review_thread_resolution": true, + "required_reviewers": [] + }, + "type": "pull_request" + }, + { + "parameters": { + "do_not_enforce_on_create": false, + "required_status_checks": [ + { + "context": "Check pull request workflow status job", + "integration_id": 15368 + } + ], + "strict_required_status_checks_policy": false + }, + "type": "required_status_checks" + }, + { + "parameters": { + "review_draft_pull_requests": true, + "review_on_push": true + }, + "type": "copilot_code_review" + } + ], + "target": "branch" +} diff --git a/repo-config/ruleset-develop.json b/repo-config/ruleset-develop.json deleted file mode 100644 index daf7dd46..00000000 --- a/repo-config/ruleset-develop.json +++ /dev/null @@ -1,45 +0,0 @@ -{ - "name": "develop", - "target": "branch", - "enforcement": "active", - "conditions": { - "ref_name": { - "include": ["refs/heads/develop"], - "exclude": [] - } - }, - "rules": [ - { "type": "deletion" }, - { "type": "non_fast_forward" }, - { "type": "required_linear_history" }, - { "type": "required_signatures" }, - { - "type": "pull_request", - "parameters": { - "allowed_merge_methods": ["squash"], - "dismiss_stale_reviews_on_push": true, - "require_code_owner_review": false, - "require_last_push_approval": false, - "required_approving_review_count": 0, - "required_review_thread_resolution": true - } - }, - { - "type": "required_status_checks", - "parameters": { - "do_not_enforce_on_create": false, - "strict_required_status_checks_policy": false, - "required_status_checks": [ - { "context": "Check pull request workflow status job", "integration_id": 15368 } - ] - } - }, - { - "type": "copilot_code_review", - "parameters": { - "review_draft_pull_requests": true, - "review_on_push": true - } - } - ] -} diff --git a/repo-config/ruleset-main.json b/repo-config/ruleset-main.json deleted file mode 100644 index 0864a0e1..00000000 --- a/repo-config/ruleset-main.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "name": "main", - "target": "branch", - "enforcement": "active", - "conditions": { - "ref_name": { - "include": ["refs/heads/main"], - "exclude": [] - } - }, - "rules": [ - { "type": "deletion" }, - { "type": "non_fast_forward" }, - { "type": "required_signatures" }, - { - "type": "pull_request", - "parameters": { - "allowed_merge_methods": ["merge"], - "dismiss_stale_reviews_on_push": true, - "require_code_owner_review": false, - "require_last_push_approval": false, - "required_approving_review_count": 0, - "required_review_thread_resolution": true - } - }, - { - "type": "required_status_checks", - "parameters": { - "do_not_enforce_on_create": false, - "strict_required_status_checks_policy": false, - "required_status_checks": [ - { "context": "Check pull request workflow status job", "integration_id": 15368 } - ] - } - }, - { - "type": "copilot_code_review", - "parameters": { - "review_draft_pull_requests": true, - "review_on_push": true - } - } - ] -} diff --git a/repo-config/settings.json b/repo-config/settings.json index fc373efd..b7f1608b 100644 --- a/repo-config/settings.json +++ b/repo-config/settings.json @@ -1,7 +1,10 @@ { - "allow_squash_merge": true, + "has_wiki": false, + "has_projects": false, "allow_merge_commit": true, + "allow_squash_merge": true, "allow_rebase_merge": false, "allow_auto_merge": true, + "allow_update_branch": true, "delete_branch_on_merge": false } diff --git a/spec/secrets.json b/spec/secrets.json new file mode 100644 index 00000000..e7088a52 --- /dev/null +++ b/spec/secrets.json @@ -0,0 +1,34 @@ +{ + "note": "Repo-scoped adaptation of the fleet hub's canonical spec/secrets.json, per the repo-config downstream carry. This repo publishes Docker images to Docker Hub (static secrets; Docker Hub has no OIDC equivalent) plus a GitHub release (no credentials), and uploads coverage to Codecov, so the fleet baseline applies plus the docker-hub and codecov mechanisms. AUDIT.md cross-checks these names (values are never read).", + "baseline": { + "requires": ["CODEGEN_APP_CLIENT_ID", "CODEGEN_APP_PRIVATE_KEY"], + "forbids": ["CODEGEN_APP_ID"], + "workflowNeeds": ["actions/create-github-app-token"], + "stores": ["actions", "dependabot"], + "note": "The App-token secrets power the App-signed merge-bot (auto-merge that re-triggers downstream workflows), which every fleet repo runs; also consumed by codegen and the upstream-version tracker where present. Used via actions/create-github-app-token with the client-id input (not the deprecated app-id). The CODEGEN_* name is historical, not codegen-specific." + }, + "mechanisms": { + "docker-hub": { + "kind": "static-secret", + "requires": ["DOCKER_HUB_USERNAME", "DOCKER_HUB_ACCESS_TOKEN"], + "forbids": [], + "workflowNeeds": [], + "stores": ["actions", "dependabot"] + }, + "codecov": { + "kind": "static-secret", + "requires": ["CODECOV_TOKEN"], + "forbids": [], + "workflowNeeds": ["codecov/codecov-action"], + "stores": ["actions"], + "note": "Coverage upload is report-only (fail_ci_if_error: false), so a Codecov hiccup never fails the gate." + } + }, + "targetMechanisms": { + "docker": "docker-hub", + "github-release": null + }, + "typeMechanisms": { + "csharp": "codecov" + } +} From d99097b9aea9d747d7fc735b99c678ba97969b91 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Fri, 17 Jul 2026 09:43:17 -0700 Subject: [PATCH 17/19] Add Regression Test Suite and Reduced-Corpus Tooling (#855) Add a standalone RegressionTests/ suite that preserves the reproducible, cross-version regression-corpus process: a ZFS-clone harness plus stdlib-only Python tooling to catalog issues, build and validate a reduced corpus behind a prove-equivalence gate, locate decode signatures, and audit physical-error-shape coverage. Wire ruff + mypy into CI (uvx, pinned) and VSCode tasks, and document the suite across the repo docs. No media or media filenames are committed: media-specific reduction rules are externalized to a file that lives with the media, and only a synthetic example ships in the repo. First Python in the repo; version unchanged (3.21). Co-Authored-By: Claude Opus 4.8 --- .github/workflows/validate-task.yml | 15 + .gitignore | 6 + .vscode/tasks.json | 59 ++- AGENTS.md | 2 + ARCHITECTURE.md | 4 + HISTORY.md | 1 + PlexCleaner.code-workspace | 8 +- README.md | 64 +-- RegressionTests/README.md | 183 +++++++++ RegressionTests/RegressionTest.sh | 393 ++++++++++++++++++ RegressionTests/audit_physical.py | 113 ++++++ RegressionTests/catalog_corpus.py | 98 +++++ RegressionTests/corpus_common.py | 364 +++++++++++++++++ RegressionTests/locate_issue.py | 221 ++++++++++ RegressionTests/pyproject.toml | 26 ++ RegressionTests/reduce_corpus.py | 402 +++++++++++++++++++ RegressionTests/reduction-rules.example.json | 16 + 17 files changed, 1910 insertions(+), 65 deletions(-) create mode 100644 RegressionTests/README.md create mode 100755 RegressionTests/RegressionTest.sh create mode 100644 RegressionTests/audit_physical.py create mode 100644 RegressionTests/catalog_corpus.py create mode 100644 RegressionTests/corpus_common.py create mode 100644 RegressionTests/locate_issue.py create mode 100644 RegressionTests/pyproject.toml create mode 100644 RegressionTests/reduce_corpus.py create mode 100644 RegressionTests/reduction-rules.example.json diff --git a/.github/workflows/validate-task.yml b/.github/workflows/validate-task.yml index a8dbd40d..d4096628 100644 --- a/.github/workflows/validate-task.yml +++ b/.github/workflows/validate-task.yml @@ -84,6 +84,21 @@ jobs: HISTORY.md incremental_files_only: false + # Lint the RegressionTests Python tooling with ruff + mypy via uvx (no project install; the + # tooling is stdlib-only). Versions are pinned here in CI (bumpable) for a reproducible gate; + # the VSCode tasks run the latest tools, so local may differ slightly by design. Config lives + # in RegressionTests/pyproject.toml, so run from that directory (mypy resolves config from CWD). + - name: Setup uv step + uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 + + - name: Lint Python step + working-directory: RegressionTests + run: | + set -euo pipefail + uvx ruff@0.15.22 check . + uvx ruff@0.15.22 format --check . + uvx mypy@2.3.0 . + - name: Lint workflows step uses: raven-actions/actionlint@3d39aea434753780c3b3d4a1a31c854b4dbf49d7 # v2.2.0 diff --git a/.gitignore b/.gitignore index 21241c2f..d0835d02 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,9 @@ coverage/ *.log *.user + +# Python tooling (RegressionTests/) +__pycache__/ +*.pyc +.mypy_cache/ +.ruff_cache/ diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 63c9bb8d..1c39f20a 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -323,6 +323,60 @@ "clear": false } }, + { + "label": "Lint: Ruff", + "type": "shell", + "command": "uvx", + "args": [ + "ruff", + "check", + "." + ], + "options": { + "cwd": "${workspaceFolder}/RegressionTests" + }, + "problemMatcher": [], + "presentation": { + "showReuseMessage": false, + "clear": false + } + }, + { + "label": "Lint: Ruff Format", + "type": "shell", + "command": "uvx", + "args": [ + "ruff", + "format", + "--check", + "." + ], + "options": { + "cwd": "${workspaceFolder}/RegressionTests" + }, + "problemMatcher": [], + "presentation": { + "showReuseMessage": false, + "clear": false + } + }, + { + "label": "Lint: Mypy", + "type": "shell", + "command": "uvx", + "args": [ + "mypy", + "." + ], + "options": { + "cwd": "${workspaceFolder}/RegressionTests" + }, + "problemMatcher": [], + "presentation": { + "showReuseMessage": false, + "clear": false + } + }, { "label": "Lint: All (CI parity)", "dependsOrder": "sequence", @@ -332,7 +386,10 @@ "Lint: EditorConfig", "Lint: Workflows", "Lint: Markdown", - "Lint: Spelling" + "Lint: Spelling", + "Lint: Ruff", + "Lint: Ruff Format", + "Lint: Mypy" ], "problemMatcher": [], "presentation": { diff --git a/AGENTS.md b/AGENTS.md index 82f6bad6..9e1cee39 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -106,6 +106,7 @@ Applies to code and workflow (`#`) comments alike. - [`.editorconfig`](./.editorconfig) is the single source of truth for line endings: CRLF for `.md`, `.cs`, XML/`.csproj`/`.props`, non-workflow `.yml`/`.yaml`, `.json`, `.cmd`/`.bat`/`.ps1`; LF for `.sh`, Dockerfiles, and workflow YAML (`.github/workflows/*.{yml,yaml}`). Workflow YAML is pinned LF because Dependabot and Actions rewrite it with LF, so declaring LF keeps it consistent instead of mixed; git still leaves endings alone (`* -text`) and CI's `editorconfig-checker` enforces it. The `[*.cs]`/ReSharper style block applies because this repo ships .NET. - **Always honor the `.editorconfig` ending.** Create a file with its spec ending; when editing a file, bring the whole file to spec (a file-wide EOL fix alongside the content change is expected, not a violation); if you come across a file with the wrong ending, fix it. [`.gitattributes`](./.gitattributes) (`* -text`) governs git's own normalization - it is not a license to leave a file on the wrong ending. Verify with `file ` after writing. +- **Python (`.py`) and `.toml` are CRLF.** They have no `[*.py]`/`[*.toml]` override, so they inherit the `[*]` CRLF default (matching the audited convention that keeps Python on the repo default rather than pinning LF). Only the `.sh` harness is LF. ### Quantitative Claims @@ -221,6 +222,7 @@ An **expected, recoverable** failure escalates through the standard repair tiers - **PlexCleanerTests** (`PlexCleanerTests/PlexCleanerTests.csproj`) - xUnit v3 test suite. Assertions via AwesomeAssertions. - **`Docker/`** - multi-arch Linux container build (`ubuntu:rolling`, `linux/amd64` + `linux/arm64`); runs as a `nonroot` user, mounts media under `/media`. +- **`RegressionTests/`** - regression harness and tooling: a ZFS-clone Bash harness plus standalone stdlib-only Python utilities (catalog / reduce / locate / audit) that verify processing decisions stay consistent across versions against a curated media collection. The Python tooling is linted with ruff and type-checked with mypy (config in `RegressionTests/pyproject.toml`); it is the only Python in the repo. No media or media filenames are committed - media-specific reduction rules live with the media as an external JSON file, and the repo ships only a synthetic example. See [`RegressionTests/README.md`](./RegressionTests/README.md). - **Build configuration**: - Common MSBuild properties (`TargetFramework`, `Nullable`, `ImplicitUsings`, `AnalysisLevel`, etc.) live in `Directory.Build.props` at the solution root. Do not duplicate these in individual `.csproj` files - only add a property to a `.csproj` when it is project-specific or overrides the shared default. - All NuGet package versions are centralised in `Directory.Packages.props`. `PackageReference` elements in `.csproj` files must not include a `Version` attribute. Asset metadata (`PrivateAssets`, `IncludeAssets`) stays in the `.csproj` `PackageReference` element. diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index ef3edc61..49451373 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -316,6 +316,10 @@ Check states with `HasFlag()`, combine with `|=` - Docker tests: Download Matroska test files from GitHub - CI: Separate workflows for build tests and Docker tests +### Regression Testing + +- Cross-version processing-consistency checks against a curated media collection, with a ZFS-clone harness and Python catalog / reduce / locate / audit tooling under `RegressionTests/`. See [`RegressionTests/README.md`](./RegressionTests/README.md). + ## Build and Release The authoritative release and workflow governance is in [AGENTS.md](./AGENTS.md). This section is a short architectural summary. diff --git a/HISTORY.md b/HISTORY.md index 9674a8bd..2d4f954a 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -42,6 +42,7 @@ Utility to optimize media files for Direct Play in Plex, Emby, Jellyfin, etc. - Handle the `SIGINT`, `SIGTERM`, and `SIGQUIT` termination signals (`docker stop`, `Ctrl+C`) so processing is interrupted gracefully and the summary and exit code are logged before exit. The custom `Ctrl+Q`/`Ctrl+Z` exit keys are removed in favor of the standard signals. - Normalize `Default` track flags instead of only warning about them: clear the flag on a lone track of a type, keep the preferred audio track as the single default when multiple are flagged, and clear all default flags on subtitle tracks. - Added a `custom` command that loads a user-provided plugin assembly implementing `IProcessPlugin` and runs it over the media files, reusing the file iteration and processing API for bespoke re-processing or repair. Includes the `MatroskaHeaderCleanup` example plugin. Not available in AOT builds. + - Added a regression test suite and reduced-corpus tooling under `RegressionTests/`: a ZFS-clone harness and Python catalog / reduce / locate / audit utilities that verify processing decisions stay consistent across versions. No application changes. - Version 3.19: - Reworked the CI/CD pipeline to a branch-scoped self-publishing model: a weekly scheduled run (and manual dispatch) publishes both `main` (stable, Docker `latest`) and `develop` (prerelease, Docker `develop`) - native executables, the multi-arch Docker image, and the GitHub release - while merges accumulate until the next run. No application changes. - Added `WORKFLOW.md` (the canonical CI/CD specification) and `repo-config/` (rulesets and repository settings as code). diff --git a/PlexCleaner.code-workspace b/PlexCleaner.code-workspace index b036bfe8..1644cde7 100644 --- a/PlexCleaner.code-workspace +++ b/PlexCleaner.code-workspace @@ -32,15 +32,19 @@ }, "extensions": { "recommendations": [ + "charliermarsh.ruff", "csharpier.csharpier-vscode", "davidanson.vscode-markdownlint", "editorconfig.editorconfig", + "fanaticpythoner.better-todo-tree", "github.vscode-github-actions", "ms-azuretools.vscode-docker", "ms-dotnettools.csdevkit", + "ms-python.mypy-type-checker", + "ms-python.python", + "ms-python.vscode-pylance", "streetsidesoftware.code-spell-checker", - "yzhang.markdown-all-in-one", - "fanaticpythoner.better-todo-tree" + "yzhang.markdown-all-in-one" ] } } diff --git a/README.md b/README.md index 1fa5b36b..159c6e78 100644 --- a/README.md +++ b/README.md @@ -936,69 +936,9 @@ docker run \ ### Regression Testing -Regression testing ensures consistent behavior across versions by comparing processing results on the same media files. +Regression testing ensures consistent behavior across versions by comparing processing results on the same media files, down to the per-file processing decision. -The behavior of the tool is very dependent on the media files being tested, and the following process can facilitate regressions testing, assuring that the process results between versions remain consistent. - -- Maintain a collection of troublesome media files that resulted in functional changes. -- Create a ZFS snapshot of the media files to test. -- Process the files, using a known good version, and save the results in JSON format using the `--resultsfile` option. -- Restore the ZFS snapshot allowing repetitive testing using the original files. -- Process the files again using the under test version. -- Compare the JSON results file from the known good version with the version under test. -- Investigate any file comparison discrepancies. - -E.g. - -```shell -# Copy troublesome files -rsync -av --delete --progress /data/media/Troublesome/. /data/media/test -chown -R nobody:users /data/media/test -chmod -R ug=rwx,o=rx /data/media/test - -# Take snapshot -zfs destroy hddpool/media/test@backup -zfs snapshot hddpool/media/test@backup -``` - -```shell -# Config -PlexCleanerApp=/PlexCleaner/Debug/PlexCleaner -MediaPath=/Test/Media -ConfigPath=/Test/Config - -# Test function -RunContainer () { - local Image=$1 - local Tag=$2 - - # Rollback to snapshot - sudo zfs rollback hddpool/media/test@backup - - # Process files - docker run \ - -it \ - --rm \ - --pull always \ - --name PlexCleaner-Test \ - --user nobody:users \ - --env TZ=America/Los_Angeles \ - --volume /data/media/test:$MediaPath:rw \ - --volume /data/media/PlexCleaner:$ConfigPath:rw \ - $Image:$Tag \ - $PlexCleanerApp process \ - --settingsfile=$ConfigPath/PlexCleaner.json \ - --logfile=$ConfigPath/PlexCleaner-$Tag.log \ - --mediafiles=$MediaPath \ - --testsnippets \ - --quickscan \ - --resultsfile=$ConfigPath/Results-$Tag.json -} - -# Test containers -RunContainer docker.io/ptr727/plexcleaner latest -RunContainer docker.io/ptr727/plexcleaner develop -``` +The behavior of the tool is very dependent on the media files being tested. A reproducible process and its tooling live under [`RegressionTests/`](./RegressionTests/): a ZFS-clone harness that processes a curated collection of troublesome media through a given image tag, plus utilities that derive a machine-readable issue catalog, build a proven-equivalent reduced collection, and audit physical-error coverage. See [`RegressionTests/README.md`](./RegressionTests/README.md) for details. ## Development Tooling diff --git a/RegressionTests/README.md b/RegressionTests/README.md new file mode 100644 index 00000000..9889549f --- /dev/null +++ b/RegressionTests/README.md @@ -0,0 +1,183 @@ +# Regression Tests + +Tooling and a reproducible process for verifying that PlexCleaner's processing decisions stay +consistent across versions, using a curated collection of troublesome media files. + +PlexCleaner's behavior depends heavily on the specific media it processes: most functional changes +are driven by a media file or media tool quirk affecting playback. This suite pins that behavior by +processing the same collection through successive builds and comparing the results down to the +per-file processing decision. + +## What is (and is not) in this directory + +The committed contents are code plus a synthetic example. No media and no media filenames live in +the repository: + +- The **media collection** lives on a server, next to the media (like a Plex library would). It is + never committed. The copyrighted filenames stay out of source control entirely. +- The **media-specific reduction rules** (issue-localized cut windows) also live with the media, in + an external JSON file the tooling reads and writes. The repo ships only + [`reduction-rules.example.json`](reduction-rules.example.json) with synthetic placeholder names. + +## Layout + +- [`RegressionTest.sh`](RegressionTest.sh) -- the harness: provision a test dataset, process it + through one Docker image tag, and write per-version results and logs for diffing. +- [`corpus_common.py`](corpus_common.py) -- shared library: log parsing, deterministic issue + classification, and the clip and metadata-surgery helpers. +- [`catalog_corpus.py`](catalog_corpus.py) -- derive a machine-readable issue catalog + (`catalog.json`) for a collection from a processing run. +- [`reduce_corpus.py`](reduce_corpus.py) -- build and validate a reduced collection: shrink each + sample while proving every issue survives. +- [`locate_issue.py`](locate_issue.py) -- find where a decode signature lives in a file, and + optionally record the located window into the external rules file. +- [`audit_physical.py`](audit_physical.py) -- physical-error-shape coverage audit of the reduced + collection. +- [`reduction-rules.example.json`](reduction-rules.example.json) -- synthetic example of the + external rules schema. +- [`pyproject.toml`](pyproject.toml) -- ruff and mypy configuration for the Python tooling. + +## The collection + +Two collections live side by side in the media dataset, each with its own generated `catalog.json`: + +- `full/` -- the complete troublesome samples. The source of truth. +- `reduced/` -- shorter clips derived from `full/`, each proven to reproduce the same issue set. A + reduced run is far faster (minutes instead of hours) and is the default for iterating. + +Both are read-only during a run. The harness never mutates the collection; it processes a +disposable copy. + +## Running the harness + +`RegressionTest.sh` provisions a disposable test dataset as a zero-copy ZFS clone of the newest +collection snapshot, processes it through a single Docker image tag, and writes results into a +directory named after the image build version so runs stay durable and version-to-version +comparable. + +```shell +sudo ./RegressionTest.sh [quick|full] [tag] [corpus] [plugin...] +``` + +- `quick` (default) keeps full-file scanning as in production but writes short test snippets to + shorten remux and re-encode. `full` also processes the complete media. +- `tag` is the Docker image tag to test (default `develop`). +- `corpus` selects `full` (default) or `reduced`. +- `plugin` names optional example plugins to build and run after processing, to confirm a plugin + loads and runs against the processed dataset. + +Provisioning uses ZFS clones rather than `rsync`: the clone is instant and drift-free, and a +rollback to the clone's own snapshot works even while long-running media containers hold the mount. + +### Full-file scanning + +The harness scans the whole file; it does not use `--quickscan`. Bounding the scan to the start of +the file causes false negatives for defects that surface later, notably closed-caption detection +and interlace detection, so a full scan is the correct default for regression comparison. + +## The issue catalog + +`catalog_corpus.py` turns a processing run into `catalog.json`: one entry per file recording its +processing State, the detections it triggered, and the classified decode-error subtypes. This +catalog is the ground truth the reduction proves against, and the artifact compared between +versions. + +Classification is deterministic and lives in `corpus_common.py`: raw ffmpeg error lines carry +per-site coordinates (macroblock positions, picture numbers) and accumulate across every corrupt +site, so they are normalized to a stable signature class before comparison. + +## Reducing the collection + +`reduce_corpus.py` shrinks each sample while proving no issue is lost. A candidate clip is processed +through the image and must match the source catalog entry on all of: + +- State equality (the processing-decision fingerprint, which catches issues that leave no log + signature). +- detections superset (every detection re-surfaces). +- verify-error signatures superset (every error class re-surfaces). + +Any miss keeps the original whole, so an issue is never dropped. + +### Cutter ladder + +Cutting a clip can silently repair the very defect the sample exists to capture, and the two cutters +have mirror-image side effects: an `mkvmerge` cut preserves timestamp defects but strips +language-IETF metadata, while an `ffmpeg` cut preserves metadata but normalizes some timestamp +defects. So the tool tries a ladder of cutters plus in-place metadata surgery and lets the +prove-equivalence gate pick the one that keeps this file's issues: + +- head clips and region clips via `mkvmerge` and `ffmpeg`. +- surgical rungs that edit the header in place with no remux: a `noietf` rung re-injects the + missing-IETF-metadata defect an `mkvmerge` cut would repair, and a `fixietf` rung sets IETF on an + `ffmpeg` cut so a timestamp defect drives the verify-and-repair chain. + +Samples at or near the window length ship verbatim, because any cut remuxes and would repair +container or metadata defects. + +### Region rules (generate on demand) + +Most defects live in the head of the file, so the default is a head clip. Defects deep in a file +need an issue-localized window. Those windows are media-specific, so they are not hard-coded; they +live in an external rules file next to the collection (default `reduction-rules.json` beside the +catalog). + +Generate them from your own media on demand: + +```shell +# locate the decode signature and record a padded window into the rules file +python3 locate_issue.py --catalog /path/to/full/catalog.json --full --write-rules /path/to/full/reduction-rules.json + +# build the reduced collection, reading those windows +python3 reduce_corpus.py --catalog /path/to/full/catalog.json --mode generate --out /path/to/reduced +``` + +The rules schema is a `regions` map keyed by source basename; see +[`reduction-rules.example.json`](reduction-rules.example.json). If the rules file is absent, every +file is head-clipped and the tool says so. + +## Physical-shape coverage audit + +The reduction gate compares broad signature classes, and a class can lump several distinct physical +ffmpeg messages together. `audit_physical.py` closes that gap: it extracts every physical error +shape (the exact message template, with run- and site-varying content normalized out) from the +ground run and from each reduced clip, and reports any source shape a clip fails to reproduce. It +augments the reduced `catalog.json` with per-file and corpus-level coverage figures, so an +under-covered area is always visible and it is known when a change warrants a full-collection run. + +## Naming conventions + +Collection filenames follow a small set of conventions so the catalog stays readable: + +- a descriptive real title for a naturally occurring sample. +- a codec-matrix name (`codec_container`) for a sample that exists to exercise a specific + combination. +- a `Word-Word` behavior name for a sample built to test one behavior. +- a `[container]` disambiguation tag appended only when two samples would otherwise collide on the + output stem (PlexCleaner renames every output to `.mkv`). +- a filename fixture whose media is a tiny synthetic clip and whose filename is the actual test. + +## Update, validate, snapshot + +The working loop when the collection changes: + +1. Update the collection (add or adjust a sample). +2. Regenerate the affected catalog with `catalog_corpus.py`. +3. Rebuild and validate the reduced collection with `reduce_corpus.py`, and audit coverage with + `audit_physical.py`. +4. Snapshot the dataset so a run can clone from it. + +## Python tooling + +The Python utilities are standalone and stdlib-only (subprocess, json, argparse, pathlib, re). They +are linted with ruff and type-checked with mypy; the configuration is in +[`pyproject.toml`](pyproject.toml). Run them via `uvx`, which needs no install: + +```shell +uvx ruff check . +uvx ruff format --check . +uvx mypy . +``` + +These run the latest tools and are available as VSCode tasks; CI pins exact versions (bumpable +there), so local results may differ slightly - by design, so local tooling never silently falls +behind. Python source is CRLF, matching the repository's default line-ending convention. diff --git a/RegressionTests/RegressionTest.sh b/RegressionTests/RegressionTest.sh new file mode 100755 index 00000000..a4eb9d13 --- /dev/null +++ b/RegressionTests/RegressionTest.sh @@ -0,0 +1,393 @@ +#!/bin/bash + +# Regression test harness: process the ZFS test dataset through a single PlexCleaner Docker +# image tag and write per-version results/logs for diffing against other runs. +# +# Usage: RegressionTest.sh [quick|full] [tag] [corpus] [plugin...] +# quick (default) full-file scan as in production, remux/re-encode to short testsnippets +# full also remux/re-encode the complete media (slowest) +# tag docker image tag to test, default develop; the runner specifies one tag per run +# corpus full (default) or reduced - which corpus subdirectory of the test clone to process +# plugin optional plugin name(s) from the registry to run after processing, default MatroskaHeaderCleanup +# +# Test data provisioning (2026-07-16): the corpus lives in the snapshotted dataset +# hddpool/media/troublesome (full/ + reduced/); each run provisions hddpool/media/test as an +# instant zero-copy ZFS CLONE of the newest corpus snapshot (no rsync, no drift - the clone +# replaces the old PrepTestDataset.sh flow). Rollbacks inside the run use the clone's @backup. +# +# Results are written into a directory named after the image build version (the "version" +# label, e.g. 3.20.7-g3235a8a055 - the same tag used for source code sync and plugin build). +# This makes results durable and version-to-version comparable, and replaces the old manual +# step of copying results into a version-named folder when happy with a release. Re-running +# the same build overwrites the files in that build's directory. +# +# The flow is: +# 1. pull the image +# 2. read the build version from the image "version" label +# 3. create the version-named directory under this folder +# 4. copy the live PlexCleaner.json settings into the directory (durable record) +# 5. write buildinfo.json (channel, version, image digest, run timestamp) +# 6. build the requested plugins from the matching source tag into the directory +# 7. run process + defaultsettings, then each plugin on the processed results, saving into the directory +# +# Settings file PlexCleaner.json is expected to use: +# "UseSystem": true, "AutoUpdate": false, "RemoveUnwantedLanguageTracks": true, +# "RemoveDuplicateTracks": true, "DeInterlace": true +# Test data provisioned per run as a ZFS clone (see above); PrepTestDataset.sh is retired. +# +# The plugin assembly is not shipped in the image, so the script clones the PlexCleaner source, +# checks out the git tag matching the image version label (the release tag), and builds the example +# MatroskaHeaderCleanup plugin from it, so the plugin API matches the running image. Requires git and +# the .NET SDK on the host; the custom run is skipped with a warning if they are missing or the version +# tag cannot be checked out. + +set -euxo pipefail + +# Run as root to allow ZFS snapshot rollback +if [[ "$(id -u)" -ne 0 ]]; then + echo "This script must be run as root" >&2 + exit 1 +fi + +# Test mode, default quick +Mode="${1:-quick}" +case "$Mode" in + quick | full) ;; + *) + echo "Usage: $0 [quick|full] [tag] [full|reduced] [plugin...]" >&2 + exit 1 + ;; +esac + +# Paths +PlexCleanerApp="/PlexCleaner/Debug/PlexCleaner" +MediaPath="/Test/Media" +ConfigPath="/Test/Config" +HostMedia="/data/media/test" +HostConfig="/data/media/PlexCleaner/RegressionTest" +CorpusDataset="hddpool/media/troublesome" +TestDataset="hddpool/media/test" +Snapshot="hddpool/media/test@backup" +Image="docker.io/ptr727/plexcleaner" +# Live master settings copied into each build's version directory +Settings="PlexCleaner.json" +# Example plugins runnable after the process test, name -> "project dll". The post-process plugin runs +# only confirm a plugin loads and runs against the processed dataset; per-plugin behaviour is verified +# in isolation elsewhere, not here. +declare -A PluginRegistry=( + [MatroskaHeaderCleanup]="Plugins/MatroskaHeaderCleanup/MatroskaHeaderCleanup.csproj MatroskaHeaderCleanup.dll" + [DtsTimestampRepair]="Plugins/DtsTimestampRepair/DtsTimestampRepair.csproj DtsTimestampRepair.dll" +) +DefaultPlugin="MatroskaHeaderCleanup" +PluginRepo="https://github.com/ptr727/PlexCleaner.git" +PluginBuildDir="/data/media/PlexCleaner/PluginBuild" + +# Parallel file-processing thread count; server has ample cores, so double the default of 4 +ThreadCount=8 + +# Process options, always run in parallel +# quick mode keeps full-file scanning but writes short testsnippets to shorten remux and re-encode +ProcessOptions=(--parallel --threadcount "$ThreadCount") +if [[ "$Mode" == "quick" ]]; then + ProcessOptions+=(--testsnippets) +fi + +# Common docker run arguments +# Allocate a TTY only when attached to one, so the script also runs non-interactively (CI, background) +TtyArg=() +[[ -t 0 ]] && TtyArg=(-it) +DockerCommon=( + "${TtyArg[@]}" + --rm + --name PlexCleaner-RegressionTest + --user nobody:users + --env TZ=America/Los_Angeles +) + +# Provision the test dataset as a zero-copy clone of the newest corpus snapshot. +# Fast path (every run): the existing clone already originates from the newest snapshot -> just +# roll back to its @backup (rollback works even while long-running media containers or SMB +# clients hold the mount in their namespaces; destroy does NOT - Plex/Jellyfin/smbd bind +# /data/media at start). +# Corpus-change path: rename the stale clone aside (rename succeeds where destroy is blocked), +# clone fresh, and opportunistically destroy any retired clones once the holders are gone. +ProvisionDataset() { + local CorpusSnap Origin + CorpusSnap="$(zfs list -H -t snapshot -o name -s creation "$CorpusDataset" | tail -1)" + if [[ -z "$CorpusSnap" ]]; then + echo "No snapshot found on $CorpusDataset - snapshot the corpus first" >&2 + exit 1 + fi + + Origin="$(zfs get -H -o value origin "$TestDataset" 2>/dev/null || true)" + if [[ "$Origin" == "$CorpusSnap" ]]; then + echo "Test clone current ($CorpusSnap) - rolling back" + zfs rollback "$Snapshot" + return + fi + + echo "Provisioning $TestDataset as a clone of $CorpusSnap" + if zfs list "$TestDataset" >/dev/null 2>&1; then + zfs rename "$TestDataset" "$TestDataset-retired-$(date +%s)" + fi + zfs clone "$CorpusSnap" "$TestDataset" + zfs snapshot "$Snapshot" + + # best-effort cleanup of retired clones (succeeds once the holders have restarted/closed) + local Retired + for Retired in $(zfs list -H -o name 2>/dev/null | grep -E "^$TestDataset-retired-" || true); do + zfs destroy -r "$Retired" 2>/dev/null && + echo "Destroyed retired clone $Retired" || + echo "Retired clone $Retired still held; will retry next run" >&2 + done +} + +# Restore the test dataset (clone) to its pristine post-provision state +RestoreDataset() { + echo "Restoring test dataset" + zfs rollback "$Snapshot" +} + +# GetImageVersion Tag -> prints the build version from the image "version" label +GetImageVersion() { + local Tag="$1" + docker image inspect "$Image:$Tag" --format '{{ index .Config.Labels "version" }}' | tr -d '\r' +} + +# WriteBuildInfo Tag Version VersionDir +# Record the channel and run metadata so results can be identified and ordered later, +# e.g. "the last develop build" vs "the last latest build". +WriteBuildInfo() { + local Tag="$1" + local Version="$2" + local VersionDir="$3" + + local ImageId Digest Created Now + ImageId="$(docker image inspect "$Image:$Tag" --format '{{ .Id }}' | tr -d '\r')" + Digest="$(docker image inspect "$Image:$Tag" --format '{{ if .RepoDigests }}{{ index .RepoDigests 0 }}{{ end }}' | tr -d '\r')" + Created="$(docker image inspect "$Image:$Tag" --format '{{ .Created }}' | tr -d '\r')" + Now="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + + cat >"$VersionDir/buildinfo.json" </dev/null || ! command -v dotnet >/dev/null; then + echo "Skipping plugin build: git and the .NET SDK are required on the host" >&2 + return 1 + fi + + echo "Building $Dll from $Image:$Tag at tag $Version" + + # Clone once, then fetch tags and check out the release tag matching the image version + if [[ ! -d "$PluginBuildDir/.git" ]]; then + git clone "$PluginRepo" "$PluginBuildDir" || return 1 + fi + git -C "$PluginBuildDir" fetch --force --tags origin || return 1 + git -C "$PluginBuildDir" checkout --force "$Version" || return 1 + + # Build the plugin (Debug matches the image binary under /PlexCleaner/Debug) + dotnet build "$PluginBuildDir/$Project" --configuration Debug || return 1 + + # Copy the built assembly into the version directory for the custom run + local Built + Built="$(find "$PluginBuildDir/.artifacts/bin" -name "$Dll" -print -quit)" + if [[ -z "$Built" ]]; then + echo "Skipping plugin build: built assembly $Dll not found" >&2 + return 1 + fi + cp "$Built" "$VersionDir/$Dll" +} + +# RunPlugin Tag Version Dll +# Run the custom command with one plugin against the current (already processed) dataset, no rollback, +# to confirm the plugin loads and runs. Per-plugin log so multiple plugin runs do not overwrite each other +RunPlugin() { + local Tag="$1" + local Version="$2" + local Dll="$3" + local ConfigDir="$ConfigPath/$Version" + [[ "$Corpus" == "reduced" ]] && ConfigDir="$ConfigPath/$Version-reduced" + + echo "Running PlexCleaner custom plugin $Dll on $Image:$Tag" + docker run \ + "${DockerCommon[@]}" \ + --volume "$HostMedia:$MediaPath:rw" \ + --volume "$HostConfig:$ConfigPath:rw" \ + "$Image:$Tag" \ + "$PlexCleanerApp" custom \ + --settingsfile="$ConfigDir/$Settings" \ + --logfile="$ConfigDir/PlexCleaner_custom_${Dll%.dll}.log" \ + "${LogOptions[@]}" \ + --mediafiles="$MediaPath/$Corpus" \ + --pluginassembly="$ConfigDir/$Dll" \ + --parallel --threadcount "$ThreadCount" +} + +# RunDefaultSettings Tag Version +RunDefaultSettings() { + local Tag="$1" + local Version="$2" + local ConfigDir="$ConfigPath/$Version" + [[ "$Corpus" == "reduced" ]] && ConfigDir="$ConfigPath/$Version-reduced" + + echo "Running PlexCleaner defaultsettings on $Image:$Tag" + docker run \ + "${DockerCommon[@]}" \ + --volume "$HostConfig:$ConfigPath:rw" \ + "$Image:$Tag" \ + "$PlexCleanerApp" defaultsettings \ + --settingsfile="$ConfigDir/PlexCleaner.defaults.json" +} + +# RunCreateSchema Tag Version +RunCreateSchema() { + local Tag="$1" + local Version="$2" + local ConfigDir="$ConfigPath/$Version" + [[ "$Corpus" == "reduced" ]] && ConfigDir="$ConfigPath/$Version-reduced" + + echo "Running PlexCleaner createschema on $Image:$Tag" + docker run \ + "${DockerCommon[@]}" \ + --volume "$HostConfig:$ConfigPath:rw" \ + "$Image:$Tag" \ + "$PlexCleanerApp" createschema \ + --schemafile="$ConfigDir/PlexCleaner.schema.json" +} + +# RunRegressionTests Tag +RunRegressionTests() { + local Tag="$1" + + docker pull "$Image:$Tag" + + # Name the results directory after the image build version (used for source sync too) + local Version + Version="$(GetImageVersion "$Tag")" + if [[ -z "$Version" ]]; then + echo "Could not read the version label from $Image:$Tag" >&2 + exit 1 + fi + echo "Build version for $Image:$Tag is $Version" + + # Create the version directory, overwriting existing files on a re-run of the same build + # reduced-corpus runs get their own results directory so they never clobber the full baseline + local VersionDir="$HostConfig/$Version" + [[ "$Corpus" == "reduced" ]] && VersionDir="$HostConfig/$Version-reduced" + mkdir -p "$VersionDir" + + # Copy the live settings into the version directory for a durable record of what was run + cp "$HostConfig/$Settings" "$VersionDir/$Settings" + + WriteBuildInfo "$Tag" "$Version" "$VersionDir" + + # v3.20+ appends to the log file by default and supports --logclear to clear on re-run + # Debug level logs each tool invocation with its command line, so a failure can be reproduced directly + local LogOptions=(--logclear --loglevel Debug) + + # Build each requested plugin from the source tag matching the image; skip any that fail to build + local Plugin Entry Project Dll + local -a BuiltPlugins=() + for Plugin in "${Plugins[@]}"; do + Entry="${PluginRegistry[$Plugin]:-}" + if [[ -z "$Entry" ]]; then + echo "Skipping unknown plugin: $Plugin" >&2 + continue + fi + read -r Project Dll <<<"$Entry" + if BuildPlugin "$Tag" "$Version" "$VersionDir" "$Project" "$Dll"; then + BuiltPlugins+=("$Dll") + else + echo "Skipping plugin run, build failed: $Plugin" >&2 + fi + done + + # The version directory was populated as root (mkdir, settings copy, buildinfo, plugin DLLs), but + # the containers run as nobody:users and write their logs and results into it; hand it to that + # user so the writes succeed regardless of the parent directory's ownership or ACLs. + chown -R nobody:users "$VersionDir" + + RunDefaultSettings "$Tag" "$Version" + # RunCreateSchema "$Tag" "$Version" + + # Provision a pristine clone of the corpus, process it, then run each built plugin on the + # processed results to confirm it loads and runs; per-plugin behaviour is verified in isolation + ProvisionDataset + # Roll the clone back to its pristine @backup even if processing or a plugin aborts under set -e, + # so a failed run never leaves the shared test dataset mutated; disarmed after the clean restore. + trap 'RestoreDataset' EXIT + RunProcess "$Tag" "$Version" + for Dll in "${BuiltPlugins[@]}"; do + RunPlugin "$Tag" "$Version" "$Dll" + done + RestoreDataset + trap - EXIT +} + +echo "Starting tests in $Mode mode" + +# Single image tag to test (default develop), corpus selection (full|reduced subdirectory of the +# test clone, default full), and optional plugins to run after processing (default the +# MatroskaHeaderCleanup example), e.g. +# RegressionTest.sh quick develop reduced DtsTimestampRepair MatroskaHeaderCleanup +Tag="${2:-develop}" +Corpus="${3:-full}" +case "$Corpus" in + full | reduced) ;; + *) + echo "Usage: $0 [quick|full] [tag] [full|reduced] [plugin...]" >&2 + exit 1 + ;; +esac +Plugins=("${@:4}") +[[ ${#Plugins[@]} -eq 0 ]] && Plugins=("$DefaultPlugin") +RunRegressionTests "$Tag" + +echo "Done with tests" diff --git a/RegressionTests/audit_physical.py b/RegressionTests/audit_physical.py new file mode 100644 index 00000000..2d10abcb --- /dev/null +++ b/RegressionTests/audit_physical.py @@ -0,0 +1,113 @@ +""" +audit_physical.py - physical-error-shape audit of the reduced corpus. + +The reduce gate compares States, detections and broad signature CLASSES; a class can lump several +distinct physical ffmpeg messages, so a clip could carry a different physical defect that maps to +the same class - and a future, more precise PlexCleaner classification would then invalidate the +sample. This audit closes that gap: it extracts every physical error SHAPE (exact message +template, run- and site-varying content normalized) from the ground run and from each reduced +clip's processing log, and reports any source shape the clip does not reproduce. + +Augments the reduced collection's catalog.json in place with per-file: + source_error_shapes / clip_error_shapes / missing_error_shapes / shape_coverage +Kept-full and verbatim entries are equal by construction (the shipped file IS the source). +""" + +import argparse +import json +from collections.abc import Iterable +from pathlib import Path + +from corpus_common import error_shape, find_run, parse_log, stem_of + +# Default paths for the reference server; override on the command line for another environment. +SCRATCH = Path("/data/media/PlexCleaner/scratch-trim") + + +def shapes_of(errors: Iterable[str]) -> list[str]: + return sorted({error_shape(e) for e in errors if e != ""}) + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--run", help="ground-truth run dir (default: newest develop run)") + ap.add_argument( + "--reduced", + default="/data/media/troublesome/reduced", + help="reduced collection dir holding catalog.json", + ) + ap.add_argument( + "--work", + default=str(SCRATCH / "work"), + help="root of the per-clip processing logs from the reduce run", + ) + args = ap.parse_args() + + reduced = Path(args.reduced) + work = Path(args.work) + run = find_run(args.run) + ground = parse_log(run / "PlexCleaner_process.log") + collection = json.loads((reduced / "catalog.json").read_text()) + manifest = collection["files"] + + gaps = 0 + audited = 0 + for e in manifest: + stem = stem_of(e["file"]) + src_shapes = shapes_of(ground.get(stem, {"errors": set()})["errors"]) + e["source_error_shapes"] = src_shapes + + if e["decision"] != "reduced" or e.get("method") == "verbatim": + # the shipped file IS the source; physically identical by construction + e["clip_error_shapes"] = src_shapes + e["missing_error_shapes"] = [] + continue + + clip_log = work / stem / "out" / "clip_process.log" + if not clip_log.exists(): + e["clip_error_shapes"] = None + e["missing_error_shapes"] = [""] + gaps += 1 + continue + audited += 1 + cl_shapes = shapes_of(parse_log(clip_log).get(stem, {"errors": set()})["errors"]) + missing = sorted(set(src_shapes) - set(cl_shapes)) + e["clip_error_shapes"] = cl_shapes + e["missing_error_shapes"] = missing + if missing: + gaps += 1 + print(f"GAP {e['file'][:66]}") + for s in missing: + print(f" - {s[:130]}") + + # Quantified completeness: the corpus does not need to be perfect, it needs to be MEASURED. + # Per-file coverage + a corpus-level figure make the residual gap explicit, so it is always + # known when a change touches an under-covered area and a full-corpus run is warranted. + total_src = total_hit = 0 + for e in manifest: + src = set(e["source_error_shapes"]) + clip = set(e["clip_error_shapes"] or []) + e["shape_coverage"] = round(len(src & clip) / len(src), 3) if src else 1.0 + total_src += len(src) + total_hit += len(src & clip) + + (reduced / "catalog.json").write_text(json.dumps(collection, indent=2, ensure_ascii=False)) + with_src = sum(1 for e in manifest if e["source_error_shapes"]) + print(f"\naudited {audited} cut clips ({with_src} files have source error shapes at all)") + print(f"files with physical-shape gaps: {gaps}") + print( + f"CORPUS PHYSICAL-SHAPE COVERAGE: {total_hit}/{total_src} shapes = " + f"{100 * total_hit / total_src:.1f}%" + if total_src + else "no source shapes" + ) + inc = [(e["file"], e["shape_coverage"]) for e in manifest if e["shape_coverage"] < 1.0] + if inc: + print("incomplete files (full-corpus run needed for changes touching these):") + for f, c in sorted(inc, key=lambda x: x[1]): + print(f" {c * 100:5.1f}% {f[:66]}") + print(f"catalog augmented: {reduced / 'catalog.json'}") + + +if __name__ == "__main__": + main() diff --git a/RegressionTests/catalog_corpus.py b/RegressionTests/catalog_corpus.py new file mode 100644 index 00000000..15c89a4f --- /dev/null +++ b/RegressionTests/catalog_corpus.py @@ -0,0 +1,98 @@ +""" +catalog_corpus.py - derive a reproducible, machine-readable issue catalog for a media corpus +from a versioned PlexCleaner regression run. + +Generated, never hand-maintained: re-run it and it re-derives every file's issue set from the +actual tool output of a chosen run. Output is `catalog.json` ONLY (automation; no human README); +it is the source of truth for what each file must reproduce and the target the reduced (quickscan) +corpus is validated against. Shared parsing/classification lives in corpus_common.py. + +Usage: catalog_corpus.py [--run ] [--out catalog.json] +""" + +import argparse +import json +import os +from collections import Counter +from pathlib import Path + +from corpus_common import ( + MEDIA_EXTS, + SRC_DIR, + buckets_for, + classify_signature, + find_run, + parse_log, + stem_of, +) + + +def build(run): + res = json.loads((run / "Results_process.json").read_text()) + versions = res.get("Versions", {}) + log = parse_log(run / "PlexCleaner_process.log") + err_files = {stem_of(x) for x in res["Results"]["Errors"]["Files"]} + vf_files = {stem_of(x) for x in res["Results"]["VerifyFailed"]["Files"]} + + entries = [] + for r in res["Results"]["Results"]: + name = os.path.basename(r["OriginalFileName"]) + if Path(name).suffix.lower() not in MEDIA_EXTS: + continue + stem = stem_of(name) + state = set(s.strip() for s in (r.get("State") or "").split(",") if s.strip()) + lg = log.get(stem, {"detections": set(), "errors": set(), "tracks": []}) + sig = classify_signature(lg["errors"]) + buckets = buckets_for(state, lg["detections"], sig, stem in err_files) + # FileDeleted / consumed samples yield no derivable output; fall back to a marker bucket + if not buckets and not state: + buckets = {"FileDeleted"} + entries.append( + { + "file": name, + "buckets": sorted(buckets), + "state": sorted(state), + "result": r.get("Result"), + "modified": r.get("Modified"), + "in_errors": stem in err_files, + "in_verifyfailed": stem in vf_files, + "detections": sorted(lg["detections"]), + "verify_errors": sorted(lg["errors"]), + "decode_subtypes": sorted(sig), + "tracks": lg["tracks"], + } + ) + entries.sort(key=lambda e: e["file"]) + return { + "schema": 1, + "source_run": run.name, + "application": versions.get("Application"), + "tools": { + t.get("ToolType", t.get("ToolFamily", "?")): t.get("Version") + for t in versions.get("Tools", []) + }, + "file_count": len(entries), + "files": entries, + } + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--run", help="version dir (default newest develop run)") + ap.add_argument("--out", default=str(SRC_DIR / "catalog.json")) + args = ap.parse_args() + + run = find_run(args.run) + catalog = build(run) + Path(args.out).write_text(json.dumps(catalog, indent=2, ensure_ascii=False)) + + print(f"run={run.name} app={catalog['application']} files={catalog['file_count']}") + print(f"catalog -> {args.out}") + bc = Counter(b for e in catalog["files"] for b in e["buckets"]) + print("bucket distribution:") + for b, n in bc.most_common(): + print(f" {n:3} {b}") + + +if __name__ == "__main__": + main() diff --git a/RegressionTests/corpus_common.py b/RegressionTests/corpus_common.py new file mode 100644 index 00000000..c48121c1 --- /dev/null +++ b/RegressionTests/corpus_common.py @@ -0,0 +1,364 @@ +""" +corpus_common.py - shared parsing + deterministic classification for the corpus tooling. + +Single source of truth so catalog_corpus.py and reduce_corpus.py never drift. Parses a +PlexCleaner process log (attributing detections/errors to files by thread affinity + line +filename) and classifies a file's State/detections/decode-signatures into a curated stable +bucket set. + +Robust to BOTH failure-log formats: a logging change inserted {Operation} before ExitCode and +appended {FileName} after the error, so a fixed regex tied to the old shape would silently drop +all errors on newer logs. +""" + +import json +import re +import sys +from pathlib import Path + +REG_DIR = Path("/data/media/PlexCleaner/RegressionTest") +SRC_DIR = Path("/data/media/troublesome/full") + +MEDIA_EXTS = { + ".mkv", + ".mp4", + ".avi", + ".wmv", + ".mov", + ".ts", + ".mpg", + ".m2ts", + ".dv", + ".webm", + ".m4v", +} + +# Robust to both log timestamp formats: +# short : "10:41:46 [INF] <9> msg" +# debug : "2026-07-15 10:41:46.977 -07:00 [INF] <9> msg" (--loglevel Debug full timestamp) +# The old short-only pattern silently matched nothing on debug logs, dropping every detection/error +# and making the equivalence check vacuous (empty >= empty). +LINE_RE = re.compile( + r"^(?:\d{4}-\d\d-\d\d )?\d\d:\d\d:\d\d(?:\.\d+)?(?: [+-]\d\d:\d\d)? \[\w+\] <(\d+)> (.*)$" +) +BEFORE_RE = re.compile(r'ProcessFiles.*?(?:Before|Skipping non-MKV file)\s*:\s*"([^"]+)"') +DETECT_RE = re.compile(r"([A-Z][A-Za-z0-9 /_-]+?) detected\b") +TRACK_RE = re.compile( + r'MkvMerge\s*:\s*(Video|Audio|Subtitle)\s*:\s*Format:\s*"([^"]*)".*?Interlaced:\s*(True|False)' +) +# old : Failed execution of FfMpeg : ExitCode: 183 : "e1 | e2 | e3" +# new : Failed execution of FfMpeg : Verify : ExitCode: 183 : "e1 | e2 | e3" : "/path/file.mkv" +FAIL_RE = re.compile(r"Failed execution of (\w+)\b.*?ExitCode:\s*-?\d+\s*(?::\s*(.*))?$") +QUOTED_RE = re.compile(r'"([^"]*)"') +# Environmental / informational stderr that is NOT a file-intrinsic issue - a reduced clip must not be +# required to reproduce these (they depend on the host, not the media). +BENIGN_NOISE = re.compile( + r"Cannot load lib|libnvidia|libcuda|Using the demultiplexer|Using the muxer for|" + r"Using the encoder|Using the decoder|Press \[q\]|configuration:|built with|" + r"deprecated pixel format", + re.IGNORECASE, +) + +DECODE_SIGNATURES = [ + ("non monotonically increasing dts", "DTS-NonMonotonic"), + ("Invalid NAL unit size", "Decode-NAL"), + ("mmco", "Decode-H264-RefPicture"), + ("reference picture missing", "Decode-H264-RefPicture"), + ("Missing reference picture", "Decode-H264-RefPicture"), + ("number of reference frames", "Decode-H264-RefFrames"), + ("cabac decode", "Decode-H264-Cabac"), + ("error while decoding", "Decode-Generic"), + # NOTE: "Invalid data found when processing input" is ffmpeg's blanket wrapper accompanying any + # hard decode failure, not a distinct issue class - classifying it forced clips to reproduce + # the wrapper severity rather than the actual signature, so it is intentionally absent. + ("noise_facs_q", "Decode-AAC"), + ("env_facs_q", "Decode-AAC"), + ("Input buffer exhausted", "Decode-AAC"), + ("Unknown subtitle segment", "Subtitle-Corrupt"), + ("quant_step_size", "TrueHD-QuantStep"), + ("Output file is empty", "Encode-EmptyOutput"), + ("non-supported file type", "Container-Unsupported"), + ("exceeds max length", "EBML-MaxLength"), +] + +STATE_BUCKET = { + "DeInterlaced": "Interlaced", + "ClearedCaptions": "ClosedCaption", + "BitrateExceeded": "Bitrate", + "SetLanguage": "Language", + "SetFlags": "Flags", + "ClearedDefaultFlags": "Flags", + "ClearedTags": "Tags", + "RemovedAttachments": "Attachments", + "RemovedCoverArt": "CoverArt", + "ReEncoded": "ReEncode", + "Repaired": "VerifyRepair", + "VerifyFailed": "VerifyFailed", + "FileReNamed": "ExtensionNormalize", +} +DETECT_BUCKET = [ + ("Interlaced", "Interlaced"), + ("Closed Caption", "ClosedCaption"), + ("Cover Art", "CoverArt"), + ("Attachment", "Attachments"), + ("language", "Language"), + ("Default flags", "Flags"), + ("flags to be set", "Flags"), + ("Tags", "Tags"), + ("Metadata", "Tags"), + ("encode", "ReEncode"), + ("Verify", "VerifyRepair"), + ("Duplicate", "DuplicateTracks"), + ("Extra video", "ExtraTracks"), +] + + +def stem_of(p): + return Path(p).stem + + +def hms(seconds): + """Seconds -> HH:MM:SS (mkvmerge rejects a seconds field > 59, so never emit 00:00:60).""" + seconds = int(seconds) + return f"{seconds // 3600:02d}:{(seconds % 3600) // 60:02d}:{seconds % 60:02d}" + + +def strip_language_ietf(path): + """Surgical defect RE-INJECTION: delete the LanguageIETF element from every track header, in + place (no remux). A mkvmerge cut writes IETF tags, repairing the 'Metadata errors' defect many + sources carry; deleting them restores the defect so the clip exercises the same repair path. + Track languages themselves are untouched.""" + import subprocess as _sp + + path = Path(path) + try: + ident = json.loads( + _sp.run( + ["mkvmerge", "-J", str(path)], stdin=_sp.DEVNULL, capture_output=True, text=True + ).stdout + ) + ntracks = len(ident.get("tracks", [])) + except Exception: + return False + if not ntracks: + return False + cmd = ["mkvpropedit", str(path)] + for i in range(1, ntracks + 1): + cmd += ["--edit", f"track:@{i}", "--delete", "language-ietf"] + r = _sp.run(cmd, stdin=_sp.DEVNULL, stdout=_sp.DEVNULL, stderr=_sp.DEVNULL) + return r.returncode == 0 + + +def set_language_ietf(path): + """Inverse surgery of strip_language_ietf: SET language-ietf on every track (from the track's + existing language), in place. An ffmpeg cut strips IETF tags, which triggers PlexCleaner's + metadata remux BEFORE verify - and that remux repairs timestamp defects the clip was built to + carry. Fixing IETF up front lets the clip enter verify metadata-clean, so the defect drives + the same verify->repair chain as the source did.""" + import subprocess as _sp + + path = Path(path) + try: + ident = json.loads( + _sp.run( + ["mkvmerge", "-J", str(path)], stdin=_sp.DEVNULL, capture_output=True, text=True + ).stdout + ) + tracks = ident.get("tracks", []) + except Exception: + return False + if not tracks: + return False + cmd = ["mkvpropedit", str(path)] + for i, t in enumerate(tracks, start=1): + lang = t.get("properties", {}).get("language", "und") or "und" + cmd += ["--edit", f"track:@{i}", "--set", f"language-ietf={lang}"] + r = _sp.run(cmd, stdin=_sp.DEVNULL, stdout=_sp.DEVNULL, stderr=_sp.DEVNULL) + return r.returncode == 0 + + +def make_head_clip(src, out, seconds, run=None, cutter="mkvmerge"): + """Cut [0, seconds] from src into out (stream copy). Returns True on a non-empty output. + + The cutters have complementary side effects, so callers try each and validate: + - mkvmerge: preserves timestamp defects and und-language, but normalizes missing IETF language + tags (repairs the "Metadata errors" defect the large samples carry). + - ffmpeg (-bitexact): preserves the missing-IETF defect, but STRIPS IETF tags from clean files + (introducing a spurious SetLanguage) and can break DTS-repair clips. + - ffmpeg-tags (no -bitexact): preserves IETF on clean files AND the metadata defect where + present, at the cost of writing Lavf writing-app tags. + All add their own track tags (ffmpeg: DURATION even with -bitexact; mkvmerge: statistics) - + the caller strips them with mkvpropedit when the source had none. Non-mkv sources always use + ffmpeg (mkvmerge cannot write their containers).""" + import subprocess as _sp + + src, out = Path(src), Path(out) + for p in out.parent.glob(out.stem + ".*"): + p.unlink() + + def _run(cmd): + if run: + return run(cmd) + return _sp.run(cmd, stdin=_sp.DEVNULL, stdout=_sp.DEVNULL, stderr=_sp.DEVNULL) + + if cutter == "mkvmerge" and src.suffix.lower() == ".mkv": + _run(["mkvmerge", "-o", str(out), "--split", f"parts:00:00:00-{hms(seconds)}", str(src)]) + if not out.exists(): + numbered = out.with_name(out.stem + "-001" + out.suffix) + if numbered.exists(): + numbered.rename(out) + else: + cmd = [ + "ffmpeg", + "-hide_banner", + "-loglevel", + "error", + "-i", + str(src), + "-t", + str(int(seconds)), + "-map", + "0", + "-c", + "copy", + "-avoid_negative_ts", + "make_zero", + ] + if cutter != "ffmpeg-tags": + cmd.append("-bitexact") + _run(cmd + [str(out)]) + return out.exists() and out.stat().st_size > 0 + + +def is_filepath_quote(q): + return (q.startswith("/") and Path(q).suffix.lower() in (MEDIA_EXTS | {".tmp"})) or bool( + re.search(r"\.tmp\d+", q) + ) + + +def extract_errors(after): + """From the text after 'ExitCode: N', return error strings (excluding the appended filename).""" + if not after: + return set() + quoted = QUOTED_RE.findall(after) + if quoted: + payload = [q for q in quoted if not is_filepath_quote(q)] + else: + payload = [re.sub(r"\s*:\s*/\S+$", "", after).strip()] + errs = set() + for p in payload: + for e in p.split(" | "): + e = e.strip() + if e and not BENIGN_NOISE.search(e): + # normalize run-varying content so identical errors compare equal across runs: + # media-root paths (/Test/Media vs /media) and ASLR pointer addresses (0x...) + e = re.sub(r"(/Test/Media|/media)/", "/", e) + e = re.sub(r"0x[0-9a-fA-F]+", "0xADDR", e) + errs.add(e) + return errs + + +def parse_log(path): + """{stem: {detections:set, errors:set, tracks:list}} by thread affinity + line filename.""" + result = {} + thread_file = {} + path = Path(path) + + def bucket(stem): + return result.setdefault(stem, {"detections": set(), "errors": set(), "tracks": []}) + + if not path.exists(): + return result + for raw in path.read_text(errors="replace").splitlines(): + m = LINE_RE.match(raw) + if not m: + continue + tid, msg = m.group(1), m.group(2) + b = BEFORE_RE.search(msg) + if b: + thread_file[tid] = stem_of(b.group(1)) + continue + stem = thread_file.get(tid) + d = DETECT_RE.search(msg) + if d: + fn = re.search(r'"([^"]+\.[A-Za-z0-9]+)"\s*$', msg) + target = stem_of(fn.group(1)) if fn else stem + if target: + bucket(target)["detections"].add(d.group(1).strip()) + continue + t = TRACK_RE.search(msg) + if t and stem: + entry = {"type": t.group(1), "format": t.group(2), "interlaced": t.group(3) == "True"} + trk = bucket(stem)["tracks"] + if entry not in trk: + trk.append(entry) + continue + f = FAIL_RE.search(msg) + if f and stem: + errs = extract_errors(f.group(2)) + bucket(stem)["errors"] |= errs if errs else {""} + return result + + +def error_shape(e): + """Reduce an (already ADDR/path-normalized) error line to its PHYSICAL SHAPE: the exact + ffmpeg message template with site-varying content (stream indexes, MB coordinates, picture + numbers, sizes) normalized out. Distinct shapes are distinct physical error identities - + far finer than the broad signature classes, and stable across corruption sites and future + PlexCleaner logic refinements.""" + s = e + # decoder/stream context brackets -> keep only the codec identity + s = re.sub(r"\[[a-z]+#\d+:\d+/(\w+) @ 0xADDR\]\s*", r"[\1] ", s) # [vist#0:0/h264 @ ..] + s = re.sub(r"\[dec:(\w+) @ 0xADDR\]\s*", r"[\1] ", s) # [dec:h264 @ ..] + s = re.sub(r"\[(\w+) @ 0xADDR\]", r"[\1]", s) # [h264 @ ..] + s = re.sub(r"\[SWR @ 0xADDR\]", "[SWR]", s) + s = re.sub(r"stream \d+", "stream N", s) + s = re.sub(r"-?\b\d+(\.\d+)?\b", "N", s) # coordinates/ids/sizes + return re.sub(r"\s+", " ", s).strip() + + +def classify_signature(errors): + subs = set() + for e in errors: + low = e.lower() + for needle, label in DECODE_SIGNATURES: + if needle.lower() in low: + subs.add(label) + break + return subs + + +def buckets_for(state, detections, sig_subtypes, in_errors): + b = set() + for flag in state: + if flag in STATE_BUCKET: + b.add(STATE_BUCKET[flag]) + for det in detections: + for needle, label in DETECT_BUCKET: + if needle.lower() in det.lower(): + b.add(label) + b |= sig_subtypes + if in_errors: + b.add("Error") + return b + + +def find_run(explicit, channel="develop"): + if explicit: + p = Path(explicit) + return p if p.is_absolute() else REG_DIR / p + best, best_mt = None, 0 + for d in REG_DIR.glob("*/"): + log, bi = d / "PlexCleaner_process.log", d / "buildinfo.json" + if not (log.exists() and bi.exists()): + continue + try: + if json.loads(bi.read_text()).get("Channel") != channel: + continue + except Exception: + continue + if log.stat().st_mtime > best_mt: + best, best_mt = d, log.stat().st_mtime + if not best: + sys.exit(f"ERROR: no {channel} run under {REG_DIR}") + return best diff --git a/RegressionTests/locate_issue.py b/RegressionTests/locate_issue.py new file mode 100644 index 00000000..3bbe19e6 --- /dev/null +++ b/RegressionTests/locate_issue.py @@ -0,0 +1,221 @@ +"""Find WHERE a localized decode signature lives, so a short issue-complete clip can be cut. + +Strategy (cheapest first): + +1. head-clip [0, W] (mkvmerge --split for mkv; ffmpeg -t copy otherwise -- reads only the head) +2. verify-decode the clip with the PlexCleaner image ffmpeg (matches the regression) and check + whether the file's catalog decode-signature substrings re-appear in stderr +3. if not reproduced in the head, a full-decode locate with -stats timestamp correlation reports + the approximate time of the first hit; ``--write-rules`` records a region window around it + +Fidelity: uses the image's ffmpeg, NOT host ffmpeg, because decode error messages differ by +version. The source corpus is READ-ONLY; clips live under scratch. +""" + +import argparse +import json +import os +import re +import subprocess +from pathlib import Path + +from corpus_common import DECODE_SIGNATURES, SRC_DIR, make_head_clip + +# Default paths for the reference server; override on the command line for another environment. +SCRATCH = Path("/data/media/PlexCleaner/scratch-trim") +IMAGE = "docker.io/ptr727/plexcleaner:develop" +WORK = SCRATCH / "locate" + +# map a catalog subtype label -> the stderr substrings that evidence it (for grep-back) +SUBTYPE_NEEDLES: dict[str, list[str]] = {} +for _needle, _label in DECODE_SIGNATURES: + SUBTYPE_NEEDLES.setdefault(_label, []).append(_needle) + +TIME_RE = re.compile(r"time=(\d+):(\d\d):(\d\d(?:\.\d+)?)") + + +def sh(cmd: list[str]) -> subprocess.CompletedProcess[str]: + return subprocess.run( + cmd, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + errors="replace", + check=False, + ) + + +# PlexCleaner's exact VerifyMedia args (no -map: default stream selection; -xerror aborts on first +# error). stats is -nostats for a plain reproduction check, -stats for timestamped locating. +def verify_args(input_path: str, stats: str = "-nostats") -> list[str]: + return [ + "-nostdin", + "-loglevel", + "error", + "-hide_banner", + stats, + "-abort_on", + "empty_output", + "-xerror", + "-fflags", + "+genpts", + "-analyzeduration", + "2G", + "-probesize", + "2G", + "-i", + input_path, + "-max_muxing_queue_size", + "1024", + "-f", + "null", + "-", + ] + + +def decode_stderr(media_dir: Path, name: str) -> str: + """Verify-decode name with the image ffmpeg using PlexCleaner's exact args; return stderr.""" + uid, gid = os.getuid(), os.getgid() + r = sh( + [ + "docker", + "run", + "--rm", + "--user", + f"{uid}:{gid}", + "--volume", + f"{media_dir}:/media:ro", + "--entrypoint", + "ffmpeg", + IMAGE, + ] + + verify_args(f"/media/{name}") + ) + return r.stdout + + +def reproduced(stderr: str, needles: list[str]) -> bool: + low = stderr.lower() + return any(n.lower() in low for n in needles) + + +def full_decode_locate(src: Path, needles: list[str]) -> float | None: + """Full-decode src (image ffmpeg + -stats, no -xerror); return the approx time (s) of the first + line matching any needle, using the nearest preceding 'time=' progress stamp. None if unseen.""" + uid, gid = os.getuid(), os.getgid() + args = [a for a in verify_args(f"/media/{src.name}", stats="-stats") if a != "-xerror"] + proc = subprocess.Popen( + [ + "docker", + "run", + "--rm", + "--user", + f"{uid}:{gid}", + "--volume", + f"{src.parent}:/media:ro", + "--entrypoint", + "ffmpeg", + IMAGE, + ] + + args, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + text=True, + errors="replace", + bufsize=1, + ) + last_t = 0.0 + hit: float | None = None + assert proc.stderr is not None + for raw in proc.stderr: + for chunk in raw.replace("\r", "\n").split("\n"): + tm = TIME_RE.search(chunk) + if tm: + last_t = int(tm.group(1)) * 3600 + int(tm.group(2)) * 60 + float(tm.group(3)) + if hit is None and any(n.lower() in chunk.lower() for n in needles): + hit = last_t + proc.wait() + return hit + + +def write_region(rules_path: Path, name: str, start: int, end: int, note: str) -> None: + """Merge a region window for name into the external rules file (create if absent).""" + data = json.loads(rules_path.read_text()) if rules_path.exists() else {} + data.setdefault("schema", 1) # keep the file self-describing, matching the shipped example + data.setdefault("regions", {})[name] = {"start": start, "end": end, "note": note} + rules_path.write_text(json.dumps(data, indent=2, ensure_ascii=False)) + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--catalog", required=True, help="source catalog.json (from catalog_corpus.py)") + ap.add_argument("-w", "--window", type=int, default=60, help="head-clip seconds") + ap.add_argument("files", nargs="*", help="basenames (default: all decode-signature files)") + ap.add_argument( + "--full", + action="store_true", + help="full-decode locate: report the approx time of the first signature hit", + ) + ap.add_argument("--write-rules", help="with --full: write located region windows to this file") + ap.add_argument( + "--pad", + type=int, + default=40, + help="seconds of padding around a located time when writing a region", + ) + args = ap.parse_args() + + cat = json.loads(Path(args.catalog).read_text()) + WORK.mkdir(parents=True, exist_ok=True) + + if args.files: + targets = [e for e in cat["files"] if e["file"] in args.files] + else: + targets = [e for e in cat["files"] if e["decode_subtypes"]] + + if args.full: + rules_path = Path(args.write_rules) if args.write_rules else None + print(f"{'FILE':52} {'SUBTYPES':28} ERROR@") + for e in targets: + name = e["file"] + src = SRC_DIR / name + subs = e["decode_subtypes"] + needles = [n for s in subs for n in SUBTYPE_NEEDLES.get(s, [])] + if not src.exists(): + print(f"{name[:52]:52} {','.join(subs)[:28]:28} MISSING SOURCE") + continue + t = full_decode_locate(src, needles) + loc = f"~{t:.0f}s ({t / 60:.1f}m)" if t is not None else "NOT FOUND in full decode" + print(f"{name[:52]:52} {','.join(subs)[:28]:28} {loc}") + if rules_path is not None and t is not None: + start = max(0, int(t) - args.pad) + write_region( + rules_path, name, start, int(t) + args.pad, f"decode signature at ~{int(t)}s" + ) + return + + print(f"{'FILE':52} {'SUBTYPES':30} {'HEAD':>6} RESULT") + for e in targets: + name = e["file"] + src = SRC_DIR / name + subs = e["decode_subtypes"] + needles = [n for s in subs for n in SUBTYPE_NEEDLES.get(s, [])] + if not src.exists(): + print(f"{name[:52]:52} {','.join(subs)[:30]:30} {'-':>6} MISSING SOURCE") + continue + wd = WORK / Path(name).stem + if wd.exists(): + for p in wd.iterdir(): + p.unlink() + wd.mkdir(parents=True, exist_ok=True) + if not make_head_clip(src, wd / name, args.window): + print(f"{name[:52]:52} {','.join(subs)[:30]:30} {'-':>6} CLIP FAILED") + continue + ok = reproduced(decode_stderr(wd, name), needles) + verdict = f"head[{args.window}s] reproduces" if ok else f"not in first {args.window}s" + print(f"{name[:52]:52} {','.join(subs)[:30]:30} {'yes' if ok else 'no':>6} {verdict}") + + +if __name__ == "__main__": + main() diff --git a/RegressionTests/pyproject.toml b/RegressionTests/pyproject.toml new file mode 100644 index 00000000..257e7cd9 --- /dev/null +++ b/RegressionTests/pyproject.toml @@ -0,0 +1,26 @@ +# Linter-only configuration for the RegressionTests Python utilities. +# +# These are standalone stdlib-only scripts (no runtime dependencies), so this file carries no +# project/build metadata - only the ruff + mypy config, mirroring the ptr727 Financial-Modeling +# conventions. Run the tools with uv (no install needed): +# +# uvx ruff check . +# uvx ruff format --check . +# uvx mypy . +# +# These run the latest tools, matching the VSCode tasks; CI (.github/workflows/validate-task.yml) +# pins exact versions (bumpable there), so local results may differ slightly - by design, so local +# tooling never silently falls behind. + +[tool.ruff] +target-version = "py313" +line-length = 100 + +[tool.ruff.lint] +extend-select = ["I"] # import sorting (isort) + +[tool.mypy] +python_version = "3.13" +warn_unused_ignores = true +warn_redundant_casts = true +no_implicit_optional = true diff --git a/RegressionTests/reduce_corpus.py b/RegressionTests/reduce_corpus.py new file mode 100644 index 00000000..3b8add7f --- /dev/null +++ b/RegressionTests/reduce_corpus.py @@ -0,0 +1,402 @@ +"""Build the reduced (quickscan) corpus: shrink samples while PROVING every issue survives. + +Each candidate clip is processed through the PlexCleaner image and must match the source's +`catalog.json` entry (generated by ``catalog_corpus.py``) on ALL of: + +- State equality (the processing-decision fingerprint; catches issues with no log signature, + e.g. timestamp-only-DTS files whose verify never logs a failed-execution line) +- detections superset (every `` detected`` from the source re-surfaces) +- verify-error signatures superset (every captured error class re-surfaces) + +Any miss -> the original is kept whole so no issue is ever lost. + +Cutting strategy per file: + +- default: head-clip ``[0, --seconds]`` (global properties + defects known to live in the head) +- region overrides (from the external rules file, located by ``locate_issue.py``): issue-localized + windows for defects deep in the file +- the cutter ladder tries mkvmerge and ffmpeg cuts plus in-place IETF surgery, because the cutters + have mirror-image side effects and only the prove-equivalence gate can pick the safe one + +The source corpus is READ-ONLY. Clips, work dirs, and outputs live under scratch / --out. + +Media-specific region windows are NOT hard-coded here (that would embed private filenames); they +live in an external rules file next to the corpus. See ``reduction-rules.example.json``. + +Modes: + +- ``validate`` (default): cut + process each clip, compare to the catalog, report PASS/FAIL +- ``generate --out DIR``: same, but on PASS write the reduced file to DIR; on FAIL copy the + original whole. Writes a reduced ``catalog.json`` with per-file decision, sizes, and issue sets. +""" + +import argparse +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path + +from corpus_common import ( + SRC_DIR, + classify_signature, + hms, + make_head_clip, + parse_log, + set_language_ietf, + stem_of, + strip_language_ietf, +) + +# Default paths for the reference server; override on the command line for another environment. +SCRATCH = Path("/data/media/PlexCleaner/scratch-trim") +IMAGE = "docker.io/ptr727/plexcleaner:develop" +APP = "/PlexCleaner/Debug/PlexCleaner" + +Region = tuple[int, int] + + +def load_regions(path: Path) -> dict[str, Region]: + """Load issue-localized cut windows from the external rules file. + + The rules live WITH the media (never in source control, to keep private filenames out of the + tree). Absent file -> no regions (head-clip everything); see ``reduction-rules.example.json`` + for the schema and ``locate_issue.py`` for generating entries. + """ + if not path.exists(): + print( + f"No rules file at {path} - regions empty (head-clip only). " + f"See reduction-rules.example.json." + ) + return {} + data = json.loads(path.read_text()) + return {name: (int(r["start"]), int(r["end"])) for name, r in data.get("regions", {}).items()} + + +def make_clip( + src: Path, out: Path, seconds: int, regions: dict[str, Region], cutter: str = "mkvmerge" +) -> bool: + """Cut a clip: a region window if the file has one in the rules, else a head-clip.""" + region = regions.get(src.name) + if not region: + return make_head_clip(src, out, seconds) + start, end = region + for p in out.parent.glob(out.stem + ".*"): + p.unlink() + if cutter == "ffmpeg": + # ffmpeg region cut gives cleaner timestamps at the cut boundary (a mkvmerge region cut can + # add a spurious verify->repaired hiccup on remux-grade HEVC), at the usual IETF-strip cost + subprocess.run( + [ + "ffmpeg", + "-hide_banner", + "-loglevel", + "error", + "-ss", + str(start), + "-i", + str(src), + "-t", + str(end - start), + "-map", + "0", + "-c", + "copy", + "-avoid_negative_ts", + "make_zero", + "-bitexact", + str(out), + ], + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + else: + # mkvmerge region cut preserves timestamp defects (ffmpeg -copyts normalizes some away) + subprocess.run( + ["mkvmerge", "-o", str(out), "--split", f"parts:{hms(start)}-{hms(end)}", str(src)], + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + if not out.exists(): + numbered = out.with_name(out.stem + "-001" + out.suffix) + if numbered.exists(): + numbered.rename(out) + return out.exists() and out.stat().st_size > 0 + + +def process_clip(workdir: Path, settings_dir: Path) -> tuple[dict, dict[str, set[str]] | None]: + """Process ``workdir/media`` through the image; return (parsed log map, {stem: state set}). + + Log + results go to ``workdir/out``, a SEPARATE mount: PlexCleaner deletes unwanted non-media + files inside the media dir, so a log written there is deleted by the very run that wrote it. + + The State map is ``None`` when the run did not complete (missing or unreadable results file), + so a failed run cannot be mistaken for an empty-State PASS and silently weaken the gate. A + completed run always writes ``clip_results.json``, even when a file's State is empty. + """ + media, out = workdir / "media", workdir / "out" + out.mkdir(parents=True, exist_ok=True) + uid, gid = os.getuid(), os.getgid() + # Capture the container output so a failed run (crash or container that never starts) can be + # diagnosed; on success it is discarded, on failure its tail is printed with the error below. + proc = subprocess.run( + [ + "docker", + "run", + "--rm", + "--user", + f"{uid}:{gid}", + "--env", + "TZ=America/Los_Angeles", + "--volume", + f"{media}:/media:rw", + "--volume", + f"{out}:/out:rw", + "--volume", + f"{settings_dir}:/config:ro", + IMAGE, + APP, + "process", + "--settingsfile=/config/PlexCleaner.json", + "--logfile=/out/clip_process.log", + "--mediafiles=/media", + "--resultsfile=/out/clip_results.json", + "--parallel", + "--testsnippets", + ], + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + errors="replace", + ) + logmap = parse_log(out / "clip_process.log") + results_file = out / "clip_results.json" + if not results_file.exists(): + tail = "\n".join((proc.stdout or "").splitlines()[-15:]) + print( + f" run did not complete (exit {proc.returncode}); no results file:\n{tail}", + file=sys.stderr, + ) + return logmap, None # run did not complete: not an empty result, a failure + try: + res = json.loads(results_file.read_text()) + results = res["Results"]["Results"] + except (json.JSONDecodeError, KeyError, OSError) as e: + print(f" results file present but unreadable: {e}", file=sys.stderr) + return logmap, None + states: dict[str, set[str]] = {} + for r in results: + states[stem_of(os.path.basename(r["OriginalFileName"]))] = { + s.strip() for s in (r.get("State") or "").split(",") if s.strip() + } + return logmap, states + + +def human(n: float) -> str: + for unit in ("B", "K", "M", "G", "T"): + if n < 1024: + return f"{n:.0f}{unit}" + n /= 1024 + return f"{n:.0f}P" + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("sources", nargs="*", help="source basenames (default: all catalog files)") + ap.add_argument("-s", "--seconds", type=int, default=60, help="head-clip length") + ap.add_argument("--catalog", required=True, help="source catalog.json (from catalog_corpus.py)") + ap.add_argument("--rules", help="external region rules file (default: next to --catalog)") + ap.add_argument( + "--settings", + default="/data/media/PlexCleaner/RegressionTest", + help="dir containing the PlexCleaner.json to process clips with", + ) + ap.add_argument("--mode", choices=["validate", "generate"], default="validate") + ap.add_argument("--out", help="output dir for the reduced set (generate mode)") + args = ap.parse_args() + + if args.mode == "generate" and not args.out: + sys.exit("generate mode requires --out DIR") + + catalog_path = Path(args.catalog) + catalog = json.loads(catalog_path.read_text()) + entries = {e["file"]: e for e in catalog["files"]} + rules_path = Path(args.rules) if args.rules else catalog_path.parent / "reduction-rules.json" + regions = load_regions(rules_path) + print(f"Catalog : {args.catalog} ({len(entries)} files)") + print(f"Head window : {args.seconds}s regions: {len(regions)} mode={args.mode}\n") + + sources = args.sources or sorted(entries) + settings_dir = Path(args.settings) + work_root = SCRATCH / "work" + work_root.mkdir(parents=True, exist_ok=True) + out_dir = Path(args.out) if args.out else None + if out_dir: + out_dir.mkdir(parents=True, exist_ok=True) + + manifest = [] + passes = fails = kept = 0 + print(f"{'FILE':54} {'SRC':>7} {'CLIP':>7} {'RATIO':>6} RESULT") + for name in sources: + e = entries.get(name) + src = SRC_DIR / name + stem = stem_of(name) + short = stem[:53] + if e is None or not src.exists(): + miss = "CATALOG" if e is None else "SOURCE" + print(f"{short:54} {'-':>7} {'-':>7} {'-':>6} MISSING {miss}") + continue + gt_state = set(e["state"]) + gt_det = set(e["detections"]) + gt_subs = set(e["decode_subtypes"]) + ssz = src.stat().st_size + + # Cutter ladder: the cutters have mirror-image side effects, so try each and let the + # prove-equivalence gate pick the one that preserves this file's issues. Surgical rungs + # (in-place header edits, no remux): "*-noietf" re-injects the missing-IETF metadata defect + # a mkvmerge cut repairs; "*-fixietf" sets IETF on an ffmpeg cut so the clip enters verify + # metadata-clean and its timestamp defect drives the verify->repair chain. + if name in regions: + ladder = ["region", "region-noietf", "region-ffmpeg", "region-ffmpeg-fixietf"] + elif src.suffix.lower() == ".mkv": + ladder = ["mkvmerge", "mkvmerge-noietf", "ffmpeg-fixietf", "ffmpeg", "ffmpeg-tags"] + else: + ladder = ["ffmpeg"] + + ok = False + attempt: dict = {} + attempts: list[dict] = [] + for cutter in ladder: + fw = work_root / stem + if fw.exists(): + shutil.rmtree(fw) + (fw / "media").mkdir(parents=True) + # cut a PRISTINE clip outside the media dir (processing mutates/renames the media copy; + # the reduced corpus must ship the unprocessed clip with its issues intact) + pristine = fw / name + base = cutter.replace("-noietf", "").replace("-fixietf", "") + if base.startswith("region"): + made = make_clip( + src, + pristine, + args.seconds, + regions, + cutter="ffmpeg" if base == "region-ffmpeg" else "mkvmerge", + ) + else: + made = make_head_clip(src, pristine, args.seconds, cutter=base) + if made and cutter.endswith("-noietf"): + made = strip_language_ietf(pristine) + elif made and cutter.endswith("-fixietf"): + made = set_language_ietf(pristine) + if not made: + attempt = {"cutter": cutter, "error": "CLIP FAILED"} + continue + # Source close to / shorter than the window -> ship VERBATIM: a cut remuxes, silently + # repairing container/metadata defects, and already-short hand-made samples need no cut. + verbatim = pristine.stat().st_size >= 0.5 * ssz + if verbatim: + shutil.copy2(src, pristine) + elif pristine.suffix.lower() == ".mkv" and "ClearedTags" not in gt_state: + # both cutters add their own track tags (ffmpeg DURATION / mkvmerge statistics); + # when the source had none, strip them in place (no remux, defects untouched) so + # the clip does not pick up a spurious ClearedTags state + subprocess.run( + ["mkvpropedit", str(pristine), "--tags", "all:"], + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + shutil.copy2(pristine, fw / "media" / name) + + logmap, states = process_clip(fw, settings_dir) + if states is None: + # processing did not complete: a failed run must not pass as an empty result + attempt = {"cutter": cutter, "error": "PROCESS FAILED"} + continue + cl = logmap.get(stem, {"detections": set(), "errors": set()}) + cl_state = states.get(stem, set()) + miss_d = gt_det - cl["detections"] + # error equivalence at the SIGNATURE-CLASS level: raw lines embed per-site coordinates + # (MB positions, picture numbers) and accumulate across every corrupt site in the + # source, which a single-site clip can never string-match + miss_e = gt_subs - classify_signature(cl["errors"]) + state_ok = cl_state == gt_state + ok = state_ok and not miss_d and not miss_e + attempt = { + "cutter": cutter, + "pristine": pristine, + "csz": pristine.stat().st_size, + "cl_state": cl_state, + "miss_d": miss_d, + "miss_e": miss_e, + "state_ok": state_ok, + "verbatim": verbatim, + } + attempts.append(attempt) + if ok or verbatim: + break # verbatim is cutter-independent; retrying cannot change it + + csz = attempt.get("csz", ssz) + ratio = f"{ssz / csz:.0f}x" + result = f"PASS ({attempt.get('cutter')})" if ok else "FAIL" + print(f"{short:54} {human(ssz):>7} {human(csz):>7} {ratio:>6} {result}") + if not ok and "error" in attempt: + print(f" {attempt['error']}") + elif not ok: + for a in attempts: # show every rung's failure so the per-cutter cause is visible + delta_plus = sorted(a["cl_state"] - gt_state) + delta_minus = sorted(gt_state - a["cl_state"]) + bits = [] + if delta_plus or delta_minus: + bits.append(f"state +{delta_plus} -{delta_minus}") + if a["miss_d"]: + bits.append(f"miss_det={sorted(a['miss_d'])}") + if a["miss_e"]: + bits.append(f"miss_sig={sorted(a['miss_e'])}") + print(f" [{a['cutter']}] {'; '.join(bits) or 'ok?'}") + + passes += ok + fails += not ok + if out_dir: + dest = out_dir / name + shutil.copy2(attempt["pristine"] if ok else src, dest) + kept += not ok + + manifest.append( + { + "file": name, + "decision": ("reduced" if ok else "kept-full") + if out_dir + else ("would-reduce" if ok else "would-keep"), + "method": ("verbatim" if attempt.get("verbatim") else attempt.get("cutter", "-")) + if ok + else "kept-full", + "source_bytes": ssz, + "clip_bytes": csz if ok else ssz, + "state_match": attempt.get("state_ok", False), + "clip_state": sorted(attempt.get("cl_state", set())), + "ground_state": sorted(gt_state), + "missing_detections": sorted(attempt.get("miss_d", set())), + "missing_signatures": sorted(attempt.get("miss_e", set())), + } + ) + + print(f"\nPASS={passes} FAIL={fails} kept-full={kept}") + mpath = (out_dir or SCRATCH) / "catalog.json" + mpath.write_text( + json.dumps( + {"schema": 1, "collection": "reduced", "file_count": len(manifest), "files": manifest}, + indent=2, + ensure_ascii=False, + ) + ) + print(f"catalog: {mpath}") + + +if __name__ == "__main__": + main() diff --git a/RegressionTests/reduction-rules.example.json b/RegressionTests/reduction-rules.example.json new file mode 100644 index 00000000..8c35d008 --- /dev/null +++ b/RegressionTests/reduction-rules.example.json @@ -0,0 +1,16 @@ +{ + "schema": 1, + "_comment": "Example media-specific reduction rules. The real file lives WITH the corpus (never in source control) so private filenames stay out of the repo. reduce_corpus.py reads the 'regions' map; locate_issue.py --write-rules generates entries by locating a decode signature in the source. Keys are source basenames as they appear in the corpus catalog.json; the names below are synthetic placeholders.", + "regions": { + "Example Show - S01E01.mkv": { + "start": 540, + "end": 620, + "note": "decode signature at ~560s, deep in the file (head-clip misses it)" + }, + "Example Movie (2020).mkv": { + "start": 1810, + "end": 1890, + "note": "interlaced-decode error near the 30m mark" + } + } +} From 9671ff21ea38650275b9afddab0048d4d14d57f4 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Fri, 17 Jul 2026 12:11:47 -0700 Subject: [PATCH 18/19] Include the Operation and File Name in Verify Failure Logs (#857) Align FfMpegTool.VerifyMedia's inline failure logging with the tool-failure convention (MediaTool.LogFailedResult), adding the operation name and file name so a verify decode-error failure is attributable to its file under parallel processing. Co-Authored-By: Claude Opus 4.8 --- PlexCleaner/FfMpegTool.cs | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/PlexCleaner/FfMpegTool.cs b/PlexCleaner/FfMpegTool.cs index 9766e4af..e626c4a3 100644 --- a/PlexCleaner/FfMpegTool.cs +++ b/PlexCleaner/FfMpegTool.cs @@ -178,23 +178,30 @@ public VerifyResult VerifyMedia(string fileName) } if (verifyResult == VerifyResult.DecodeError) { - // Log the unique error lines, a silent non-zero exit has none so omit the empty field + // Log the unique error lines, a silent non-zero exit has none so omit the empty field. + // Include the operation and file name to match the tool-failure logging convention + // (see MediaTool.LogFailedResult); VerifyMedia streams stderr so it logs inline rather + // than through that helper. string error = CleanForLog(string.Join(" | ", classifier.Errors)); if (string.IsNullOrEmpty(error)) { Log.Error( - "Failed execution of {ToolType} : ExitCode: {ExitCode}", + "Failed execution of {ToolType} : {Operation:l} : ExitCode: {ExitCode} : {FileName}", GetToolType(), - exitCode + nameof(VerifyMedia), + exitCode, + fileName ); } else { Log.Error( - "Failed execution of {ToolType} : ExitCode: {ExitCode} : {Error}", + "Failed execution of {ToolType} : {Operation:l} : ExitCode: {ExitCode} : {Error} : {FileName}", GetToolType(), + nameof(VerifyMedia), exitCode, - error + error, + fileName ); } } From 54bf89f22a60a5030573b573bd24614d44d441c2 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Fri, 17 Jul 2026 12:24:23 -0700 Subject: [PATCH 19/19] Address Promotion Review Nits: Self-Contained Comment and Clearer Local (#858) Drop the cross-project reference from RegressionTests/pyproject.toml's header comment, and deconstruct FfProbeTool.GetPackets's tuple result so the local no longer shadows the error out-parameter. No behavior change. Co-Authored-By: Claude Opus 4.8 --- PlexCleaner/FfProbeTool.cs | 8 ++++---- RegressionTests/pyproject.toml | 3 +-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/PlexCleaner/FfProbeTool.cs b/PlexCleaner/FfProbeTool.cs index 67e6e84d..c30a59b8 100644 --- a/PlexCleaner/FfProbeTool.cs +++ b/PlexCleaner/FfProbeTool.cs @@ -60,16 +60,16 @@ public bool GetPackets( [CallerMemberName] string operation = "" ) { - // Wrap async function in a task - (bool result, string error) result = GetPacketsAsync( + // Run the async worker synchronously; deconstruct so the tuple does not shadow the error out-param + (bool ok, string packetError) = GetPacketsAsync( command, async packet => await Task.FromResult(packetFunc(packet)), operation ) .GetAwaiter() .GetResult(); - error = result.error; - return result.result; + error = packetError; + return ok; } public async Task<(bool result, string error)> GetPacketsAsync( diff --git a/RegressionTests/pyproject.toml b/RegressionTests/pyproject.toml index 257e7cd9..411b6902 100644 --- a/RegressionTests/pyproject.toml +++ b/RegressionTests/pyproject.toml @@ -1,8 +1,7 @@ # Linter-only configuration for the RegressionTests Python utilities. # # These are standalone stdlib-only scripts (no runtime dependencies), so this file carries no -# project/build metadata - only the ruff + mypy config, mirroring the ptr727 Financial-Modeling -# conventions. Run the tools with uv (no install needed): +# project/build metadata - only the ruff + mypy config. Run the tools with uv (no install needed): # # uvx ruff check . # uvx ruff format --check .