From 86123dcda51a19a20a401492fb051791186b23dc Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Sun, 19 Jul 2026 07:33:02 -0700 Subject: [PATCH 1/6] Switch runtime metrics to operation-weighted progress Replace the in-tool partial-credit progress with a simpler operation-counting model: each heavy full-file operation adds the file's size to the work total when it starts and to the completed total when it ends, so progress reflects the actual, non-deterministic work with no per-file fraction weighting and no parsing of tool progress output. - Metrics: work.total/work.completed byte counters incremented at operation boundaries via an ambient thread-local file size; drop the FileSink partial-credit machinery. bytes.total stays the input-size reference. - Bracket the heavy full-file operations (closed-caption scan, idet, ffmpeg re-encode, HandBrake deinterlace, packet analysis, verify) with OpStarted/OpCompleted. - Keep the tool progress-parsing capability (ParseProgressFraction, ExecuteStreamStdOut, the Progress()/Json() builders) and its unit tests unused by this feature, for the media-tools library. - NativeAOT: enable EventSourceSupport when PublishAot is set so dotnet-counters and dotnet-monitor can read the meter from an AOT build; default builds are unchanged. Co-Authored-By: Claude Opus 4.8 --- HISTORY.md | 4 +- PlexCleaner/FfMpegTool.cs | 48 ++++++---------- PlexCleaner/FfProbeTool.cs | 10 +++- PlexCleaner/HandBrakeTool.cs | 20 ++----- PlexCleaner/Metrics.cs | 96 +++++++++++--------------------- PlexCleaner/PlexCleaner.csproj | 4 ++ PlexCleaner/Process.cs | 5 -- PlexCleaner/ProcessDriver.cs | 5 +- PlexCleaner/ProcessFile.cs | 15 ----- PlexCleanerTests/MetricsTests.cs | 90 ++++++++++++------------------ README.md | 4 +- 11 files changed, 109 insertions(+), 192 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index 96850e15..4d3b8961 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -6,8 +6,8 @@ Utility to optimize media files for Direct Play in Plex, Emby, Jellyfin, etc. - Version 3.22: - Added always-on runtime metrics published via `System.Diagnostics.Metrics` under the `PlexCleaner.Process` meter, readable with `dotnet-counters` with no extra infrastructure. - - Overall progress is weighted by input bytes rather than file count, so a run mixing tiny and huge files reports the actual work completed, and it advances smoothly during a long operation because each in-flight file contributes partial credit from the running tool (the full-file scan, the re-encode, and the deinterlace). - - Instruments include the run file and byte totals, in-flight and active-thread counts, completed bytes, the weighted `progress.ratio` and `eta.seconds`, cumulative per-outcome counters (completed, modified, errors, verify-failed, and a per-`State`-flag tally), and the `file.duration` and per-tool `tool.duration` histograms. Metrics are aggregate only, with bounded `state` and `tool` tags and no filename tags. + - Overall progress is operation-weighted: each heavy full-file operation (the closed-caption and interlace scans, bitrate analysis, re-encode, deinterlace, and verify) counts the file's size as work to do when it starts and as work done when it finishes, so a run mixing tiny and huge files reports the actual work completed, and the total grows as the non-deterministic per-file path is discovered. + - Instruments include the run file total and input byte total, in-flight and active-thread counts, the `work.total`/`work.completed` byte counters and the `progress.ratio` and `eta.seconds` derived from them, cumulative per-outcome counters (completed, modified, errors, verify-failed, and a per-`State`-flag tally), and the `file.duration` and per-tool `tool.duration` histograms. Metrics are aggregate only, with bounded `state` and `tool` tags and no filename tags. - The run-scoped gauges reset at the start of every processing pass, so monitor mode and back-to-back commands each report their own run, while the counters stay cumulative for rate display. - Instruments are inert until a listener observes them, so the feature is always on with no configuration flag and negligible idle overhead. - The meter is OpenTelemetry and `dotnet-monitor` compatible, and the Docker image ships a `counters` wrapper, so reading the metrics is `docker exec counters`. diff --git a/PlexCleaner/FfMpegTool.cs b/PlexCleaner/FfMpegTool.cs index e7f31695..806ae719 100644 --- a/PlexCleaner/FfMpegTool.cs +++ b/PlexCleaner/FfMpegTool.cs @@ -164,7 +164,10 @@ public VerifyResult VerifyMedia(string fileName) // Execute command: ffmpeg can exit 0 yet report stream errors on stderr // Classify stderr line by line as it streams to keep memory bounded, e.g. non-monotonic-DTS file emits a warning per packet VerifyClassifier.Accumulator classifier = new(); - if (!ExecuteStreamStdErr(command, classifier.Add, out int exitCode)) + Metrics.OpStarted(); + bool executed = ExecuteStreamStdErr(command, classifier.Add, out int exitCode); + Metrics.OpCompleted(); + if (!executed) { // Process could not run return VerifyResult.DecodeError; @@ -310,24 +313,6 @@ when durationUs > 0 }; } - private bool ExecuteEncodeWithProgress(Command command, string inputName) - { - Metrics.FileSink? sink = Metrics.CurrentFileSink; - return ExecuteStreamStdOut( - command, - line => - { - double? fraction = ParseProgressFraction(line, sink?.DurationUs ?? 0); - if (fraction.HasValue) - { - Metrics.ReportFileFraction(sink, fraction.Value); - } - }, - out int exitCode, - out string standardError - ) && (exitCode == 0 || LogFailedResult(exitCode, standardError, inputName)); - } - public bool ConvertToMkv( string inputName, SelectMediaProps? selectMediaProps, @@ -351,10 +336,7 @@ string outputName // Build command line Command command = GetBuilder() .GlobalOptions(options => - options - .Default() - .Progress() - .Add(Program.Config.ConvertOptions.FfMpegOptions.Global) + options.Default().Add(Program.Config.ConvertOptions.FfMpegOptions.Global) ) .InputOptions(options => options.Default().TestSnippets().InputFile(inputName)) .OutputOptions(options => @@ -368,7 +350,10 @@ string outputName .Build(); // Execute command - return ExecuteEncodeWithProgress(command, inputName); + Metrics.OpStarted(); + bool executed = Execute(command, true, true, out BufferedCommandResult result); + Metrics.OpCompleted(); + return executed && (result.ExitCode == 0 || LogFailedResult(result, inputName)); } public bool ConvertToMkv(string inputName, string outputName) @@ -379,10 +364,7 @@ public bool ConvertToMkv(string inputName, string outputName) // Build command line Command command = GetBuilder() .GlobalOptions(options => - options - .Default() - .Progress() - .Add(Program.Config.ConvertOptions.FfMpegOptions.Global) + options.Default().Add(Program.Config.ConvertOptions.FfMpegOptions.Global) ) .InputOptions(options => options.Default().TestSnippets().InputFile(inputName)) .OutputOptions(options => @@ -398,7 +380,10 @@ public bool ConvertToMkv(string inputName, string outputName) .Build(); // Execute command - return ExecuteEncodeWithProgress(command, inputName); + Metrics.OpStarted(); + bool executed = Execute(command, true, true, out BufferedCommandResult result); + Metrics.OpCompleted(); + return executed && (result.ExitCode == 0 || LogFailedResult(result, inputName)); } public bool SetTimestamps(string inputName, string outputName) @@ -525,7 +510,10 @@ public bool GetIdetText(string fileName, out string text) .Build(); // Execute command - if (!Execute(command, true, true, out BufferedCommandResult result)) + Metrics.OpStarted(); + bool executed = Execute(command, true, true, out BufferedCommandResult result); + Metrics.OpCompleted(); + if (!executed) { return false; } diff --git a/PlexCleaner/FfProbeTool.cs b/PlexCleaner/FfProbeTool.cs index 0fb1c12d..791a835f 100644 --- a/PlexCleaner/FfProbeTool.cs +++ b/PlexCleaner/FfProbeTool.cs @@ -226,7 +226,10 @@ public bool GetClosedCaptions(string fileName, out bool hasClosedCaptions) .Build(); // Execute command - if (!Execute(command, false, true, out BufferedCommandResult result)) + Metrics.OpStarted(); + bool executed = Execute(command, false, true, out BufferedCommandResult result); + Metrics.OpCompleted(); + if (!executed) { return false; } @@ -313,7 +316,10 @@ bool quickScan .Build(); // Get packet list - if (!GetPackets(command, packetFunc, out string error)) + Metrics.OpStarted(); + bool got = GetPackets(command, packetFunc, out string error); + Metrics.OpCompleted(); + if (!got) { Log.Error("Failed to get analysis packets : {FileName}", fileName); LogErrorOutput(error); diff --git a/PlexCleaner/HandBrakeTool.cs b/PlexCleaner/HandBrakeTool.cs index 85db5f25..edd57558 100644 --- a/PlexCleaner/HandBrakeTool.cs +++ b/PlexCleaner/HandBrakeTool.cs @@ -121,7 +121,7 @@ bool deInterlace // Build command line Command command = GetBuilder() - .GlobalOptions(options => options.Default().Json()) + .GlobalOptions(options => options.Default()) .InputOptions(options => options.InputFile(inputName).TestSnippets()) .OutputOptions(options => options @@ -140,20 +140,10 @@ bool deInterlace .Build(); // Execute command - Metrics.FileSink? sink = Metrics.CurrentFileSink; - return ExecuteStreamStdOut( - command, - line => - { - double? fraction = ParseProgressFraction(line); - if (fraction.HasValue) - { - Metrics.ReportFileFraction(sink, fraction.Value); - } - }, - out int exitCode, - out string standardError - ) && (exitCode == 0 || LogFailedResult(exitCode, standardError, inputName)); + Metrics.OpStarted(); + bool executed = Execute(command, true, true, out BufferedCommandResult result); + Metrics.OpCompleted(); + return executed && (result.ExitCode == 0 || LogFailedResult(result, inputName)); } [GeneratedRegex( diff --git a/PlexCleaner/Metrics.cs b/PlexCleaner/Metrics.cs index 49940233..67872c7f 100644 --- a/PlexCleaner/Metrics.cs +++ b/PlexCleaner/Metrics.cs @@ -1,4 +1,3 @@ -using System.Collections.Concurrent; using System.Diagnostics; using System.Diagnostics.Metrics; @@ -6,27 +5,6 @@ namespace PlexCleaner; internal static class Metrics { - // Per-file partial credit for the in-flight fold. - internal sealed class FileSink - { - internal long Bytes { get; init; } - - // Written on the worker thread and read on the tool pipe thread, so accessed via Volatile. - private long _durationUs; - - internal long DurationUs => Volatile.Read(ref _durationUs); - - internal void SetDurationUs(long durationUs) => Volatile.Write(ref _durationUs, durationUs); - - // Fraction as a 0..10000 permyriad so it reads and writes atomically as an int. - private int _permyriad; - - internal double Fraction => Volatile.Read(ref _permyriad) / 10000.0; - - internal void SetFraction(double fraction) => - Volatile.Write(ref _permyriad, (int)Math.Clamp(fraction * 10000.0, 0.0, 10000.0)); - } - private static readonly Meter s_meter = new("PlexCleaner.Process"); // Cumulative for the process lifetime, not reset between runs. @@ -66,13 +44,15 @@ internal void SetFraction(double fraction) => // All access is via Interlocked so the parallel loop needs no lock. private static long s_runFilesTotal; private static long s_runBytesTotal; - private static long s_runBytesCompleted; + + // Operation-weighted progress: each heavy full-file operation adds the file size to the work total when it starts and the completed total when it ends. + private static long s_runWorkTotal; + private static long s_runWorkCompleted; private static long s_runInflight; private static long s_runStartTimestamp; - // The current file is exposed via ThreadLocal so a worker-thread call site can capture it for a tool closure. - private static readonly ConcurrentDictionary s_runInflightSinks = new(); - private static readonly ThreadLocal s_currentFileSink = new(); + // The current file's size, set on the worker thread so OpStarted and OpCompleted can weight by it. + private static readonly ThreadLocal s_currentFileSize = new(); // Each State flag (minus None) with its tag, pre-built once so RecordStates allocates nothing per file. private static readonly ( @@ -119,15 +99,21 @@ static Metrics() description: "Sum of input sizes in the current run" ); _ = s_meter.CreateObservableGauge( - "plexcleaner.bytes.completed", - () => Interlocked.Read(ref s_runBytesCompleted), + "plexcleaner.work.total", + () => Interlocked.Read(ref s_runWorkTotal), + unit: "By", + description: "Operation work discovered, file size added per heavy operation, grows as the path unfolds" + ); + _ = s_meter.CreateObservableGauge( + "plexcleaner.work.completed", + () => Interlocked.Read(ref s_runWorkCompleted), unit: "By", - description: "Bytes of finished files (no partial credit for in-flight files)" + description: "Operation work finished, file size added per completed heavy operation" ); _ = s_meter.CreateObservableGauge( "plexcleaner.progress.ratio", ComputeProgress, - description: "Byte-weighted overall progress [0..1]" + description: "Operation-weighted overall progress [0..1]" ); _ = s_meter.CreateObservableGauge( "plexcleaner.eta.seconds", @@ -142,50 +128,34 @@ internal static void BeginRun(long totalFiles, long totalBytes) { _ = Interlocked.Exchange(ref s_runFilesTotal, totalFiles); _ = Interlocked.Exchange(ref s_runBytesTotal, totalBytes); - _ = Interlocked.Exchange(ref s_runBytesCompleted, 0); + _ = Interlocked.Exchange(ref s_runWorkTotal, 0); + _ = Interlocked.Exchange(ref s_runWorkCompleted, 0); _ = Interlocked.Exchange(ref s_runInflight, 0); _ = Interlocked.Exchange(ref s_runStartTimestamp, Stopwatch.GetTimestamp()); - s_runInflightSinks.Clear(); } internal static void FileStarted(long sizeBytes) { - FileSink sink = new() { Bytes = sizeBytes }; - _ = s_runInflightSinks.TryAdd(sink, 0); - s_currentFileSink.Value = sink; + s_currentFileSize.Value = sizeBytes; _ = Interlocked.Increment(ref s_runInflight); } - // Remove the current file's partial credit before FileCompleted folds its whole size. internal static void FileInflightDone() { - FileSink? sink = s_currentFileSink.Value; - s_currentFileSink.Value = null; - if (sink != null) - { - _ = s_runInflightSinks.TryRemove(sink, out _); - } + s_currentFileSize.Value = 0; _ = Interlocked.Decrement(ref s_runInflight); } - internal static FileSink? CurrentFileSink => s_currentFileSink.Value; + // A heavy full-file operation started, count the current file's size as work to do. + internal static void OpStarted() => + Interlocked.Add(ref s_runWorkTotal, s_currentFileSize.Value); - internal static void SetCurrentFileDurationUs(long durationUs) - { - if (s_currentFileSink.Value is { } sink) - { - sink.SetDurationUs(durationUs); - } - } - - // Safe to call from a tool's pipe thread. - internal static void ReportFileFraction(FileSink? sink, double fraction) => - sink?.SetFraction(fraction); + // The heavy operation finished, count the same size as work done. + internal static void OpCompleted() => + Interlocked.Add(ref s_runWorkCompleted, s_currentFileSize.Value); - // A finished file credits its whole size and its wall-clock time. - internal static void FileCompleted(long sizeBytes, TimeSpan wall) + internal static void FileCompleted(TimeSpan wall) { - _ = Interlocked.Add(ref s_runBytesCompleted, sizeBytes); s_filesCompleted.Add(1); s_fileDuration.Record(wall.TotalMilliseconds); } @@ -212,23 +182,19 @@ internal static void RecordToolDuration(MediaTool.ToolType tool, double millisec internal static void Dispose() { - s_currentFileSink.Dispose(); + s_currentFileSize.Dispose(); s_meter.Dispose(); } - // Completed bytes plus each in-flight file's partial credit, byte-weighted, guarding a zero total. + // Completed operation work over discovered operation work, guarding a zero total. internal static double ComputeProgress() { - long total = Interlocked.Read(ref s_runBytesTotal); + long total = Interlocked.Read(ref s_runWorkTotal); if (total <= 0) { return 0.0; } - double completed = Interlocked.Read(ref s_runBytesCompleted); - foreach (KeyValuePair entry in s_runInflightSinks) - { - completed += entry.Key.Bytes * entry.Key.Fraction; - } + double completed = Interlocked.Read(ref s_runWorkCompleted); return Math.Clamp(completed / total, 0.0, 1.0); } diff --git a/PlexCleaner/PlexCleaner.csproj b/PlexCleaner/PlexCleaner.csproj index 7fe9ee9f..e72a9f29 100644 --- a/PlexCleaner/PlexCleaner.csproj +++ b/PlexCleaner/PlexCleaner.csproj @@ -35,6 +35,10 @@ $(DefineConstants);PLUGINS + + + true + True diff --git a/PlexCleaner/Process.cs b/PlexCleaner/Process.cs index ab688c04..873022b6 100644 --- a/PlexCleaner/Process.cs +++ b/PlexCleaner/Process.cs @@ -152,11 +152,6 @@ out string? failedOperation break; } - // Seed the ffmpeg progress denominator now that the duration is known - Metrics.SetCurrentFileDurationUs( - (long)processFile.FfProbeProps.Duration.TotalMicroseconds - ); - // ReMux non-MKV containers using MKV file extensions // Conditional on ReMux option, fails if not Matroska and ReMux is not enabled operation = nameof(processFile.RemuxNonMkvContainer); diff --git a/PlexCleaner/ProcessDriver.cs b/PlexCleaner/ProcessDriver.cs index 8a828c0c..5c030bfc 100644 --- a/PlexCleaner/ProcessDriver.cs +++ b/PlexCleaner/ProcessDriver.cs @@ -118,8 +118,7 @@ Func taskFunc // Process all files in parallel int totalCount = fileList.Count; - // Sum sizes up front and credit the same size at completion, so a mid-run rename cannot drift the total. - // Missing files weight as zero. + // Sum input sizes up front for bytes.total and to weight each file's operations, a missing file counts as zero. Dictionary fileSizes = new(totalCount, StringComparer.Ordinal); long totalBytes = 0; foreach (string file in fileList) @@ -236,7 +235,7 @@ Func taskFunc Interlocked.Increment(ref processedCount), totalCount ); - Metrics.FileCompleted(fileSize, taskElapsed); + Metrics.FileCompleted(taskElapsed); Log.Information( "{TaskName} ({Processed:F2}%) Elapsed : {Elapsed:l} : After : {FileName}", taskName, diff --git a/PlexCleaner/ProcessFile.cs b/PlexCleaner/ProcessFile.cs index 8dae98b9..13418545 100644 --- a/PlexCleaner/ProcessFile.cs +++ b/PlexCleaner/ProcessFile.cs @@ -2551,11 +2551,6 @@ out DtsInfo? dtsInfo ); DtsInfo packetDts = new(); - // Capture the sink here so the pipe-thread packet lambda can report pts over duration. - Metrics.FileSink? sink = Metrics.CurrentFileSink; - double durationSeconds = FfProbeProps.Duration.TotalSeconds; - double maxPtsSeconds = 0; - if ( !Tools.FfProbe.GetAnalysisPackets( FileInfo.FullName, @@ -2563,16 +2558,6 @@ out DtsInfo? dtsInfo { packetBitrate.Add(packet); packetDts.Add(packet); - // Track the max pts so reordered packet timestamps only move the fraction forward. - if ( - durationSeconds > 0 - && double.IsFinite(packet.PtsTime) - && packet.PtsTime > maxPtsSeconds - ) - { - maxPtsSeconds = packet.PtsTime; - Metrics.ReportFileFraction(sink, maxPtsSeconds / durationSeconds); - } return true; }, quickScan diff --git a/PlexCleanerTests/MetricsTests.cs b/PlexCleanerTests/MetricsTests.cs index 339aec63..fbd38d33 100644 --- a/PlexCleanerTests/MetricsTests.cs +++ b/PlexCleanerTests/MetricsTests.cs @@ -10,17 +10,6 @@ namespace PlexCleanerTests; [Collection("Sequential")] public class MetricsTests { - [Fact] - public void ComputeProgress_IsByteWeightedNotCountWeighted() - { - // Two files totalling 1000 bytes; completing only the 900-byte one is 90% of the work, not - // 50% of the file count. - Metrics.BeginRun(2, 1000); - Metrics.FileCompleted(900, TimeSpan.Zero); - - _ = Metrics.ComputeProgress().Should().BeApproximately(0.9, 1e-9); - } - [Fact] public void ComputeProgress_ZeroTotal_IsZero() { @@ -29,15 +18,6 @@ public void ComputeProgress_ZeroTotal_IsZero() _ = Metrics.ComputeProgress().Should().Be(0.0); } - [Fact] - public void ComputeProgress_AllBytesDone_IsOne() - { - Metrics.BeginRun(1, 500); - Metrics.FileCompleted(500, TimeSpan.Zero); - - _ = Metrics.ComputeProgress().Should().Be(1.0); - } - [Fact] public void ComputeEtaSeconds_NoProgress_IsZero() { @@ -49,13 +29,20 @@ public void ComputeEtaSeconds_NoProgress_IsZero() [Fact] public void ComputeEtaSeconds_PartialProgress_IsFiniteAndNonNegative() { + // Discover two operations and finish one, so progress is partial and ETA is finite Metrics.BeginRun(2, 1000); - Metrics.FileCompleted(400, TimeSpan.Zero); + Metrics.FileStarted(500); + Metrics.OpStarted(); + Metrics.OpCompleted(); + Metrics.OpStarted(); double eta = Metrics.ComputeEtaSeconds(); _ = double.IsFinite(eta).Should().BeTrue(); _ = eta.Should().BeGreaterThanOrEqualTo(0.0); + + Metrics.OpCompleted(); + Metrics.FileInflightDone(); } [Fact] @@ -104,13 +91,16 @@ public void Instruments_AreObservableViaMeterListener() ); listener.Start(); - // One file completes at 400 bytes and a second stays in flight, so byte-weighted progress is 0.4 + // File 1 runs and completes one op, file 2 starts an op and stays in flight, so operation-weighted progress is 400 of 1000 bytes Metrics.BeginRun(2, 1000); Metrics.FileStarted(400); + Metrics.OpStarted(); + Metrics.OpCompleted(); Metrics.FileInflightDone(); - Metrics.FileCompleted(400, TimeSpan.Zero); + Metrics.FileCompleted(TimeSpan.Zero); Metrics.RecordStates(SidecarFile.StatesType.ReMuxed | SidecarFile.StatesType.Verified); Metrics.FileStarted(600); + Metrics.OpStarted(); listener.RecordObservableInstruments(); // The counter fired one measurement per set flag with the state tag @@ -122,7 +112,7 @@ .. longs ]; _ = states.Should().BeEquivalentTo(["ReMuxed", "Verified"]); - // One of two started files is still in flight, and progress is byte-weighted + // One of two started files is still in flight, and progress is operation-weighted _ = longs .Should() .ContainSingle(m => m.Name == "plexcleaner.files.inflight") @@ -134,55 +124,49 @@ .. longs .Which.Value.Should() .BeApproximately(0.4, 1e-9); - // Clear the in-flight file's thread-local sink + // Clear the in-flight file + Metrics.OpCompleted(); Metrics.FileInflightDone(); } [Fact] - public void ComputeProgress_FoldsInflightPartialCredit() + public void ComputeProgress_IsCompletedOverDiscoveredWork() { - Metrics.BeginRun(2, 1000); - Metrics.FileStarted(600); - Metrics.FileSink? sink = Metrics.CurrentFileSink; + // Progress is completed operation work over discovered operation work, each op weighted by file size + Metrics.BeginRun(1, 100_000); + Metrics.FileStarted(100_000); - // 600 bytes at half done is 30% of the 1000-byte run - Metrics.ReportFileFraction(sink, 0.5); - _ = Metrics.ComputeProgress().Should().BeApproximately(0.3, 1e-9); + // No operations yet, guard the zero total + _ = Metrics.ComputeProgress().Should().Be(0.0); - Metrics.ReportFileFraction(sink, 1.0); - _ = Metrics.ComputeProgress().Should().BeApproximately(0.6, 1e-9); + // First op started but not done, counted in the total only + Metrics.OpStarted(); + _ = Metrics.ComputeProgress().Should().Be(0.0); - // Finishing removes the partial credit and credits the whole size, same result here - Metrics.FileInflightDone(); - Metrics.FileCompleted(600, TimeSpan.Zero); - _ = Metrics.ComputeProgress().Should().BeApproximately(0.6, 1e-9); - } + // First op done, all discovered work is complete + Metrics.OpCompleted(); + _ = Metrics.ComputeProgress().Should().Be(1.0); - [Fact] - public void ReportFileFraction_ClampsOutOfRange() - { - Metrics.BeginRun(1, 1000); - Metrics.FileStarted(1000); - Metrics.FileSink? sink = Metrics.CurrentFileSink; + // A second op is discovered, the total grows and the ratio dips + Metrics.OpStarted(); + _ = Metrics.ComputeProgress().Should().BeApproximately(0.5, 1e-9); - Metrics.ReportFileFraction(sink, 1.5); + Metrics.OpCompleted(); _ = Metrics.ComputeProgress().Should().Be(1.0); - Metrics.ReportFileFraction(sink, -0.5); - _ = Metrics.ComputeProgress().Should().Be(0.0); - Metrics.FileInflightDone(); } [Fact] - public void FileInflightDone_RemovesPartialCredit() + public void OpAfterInflightDone_CountsNothing() { + // FileInflightDone clears the current file size, so a stray late op adds no work Metrics.BeginRun(1, 1000); Metrics.FileStarted(400); - Metrics.ReportFileFraction(Metrics.CurrentFileSink, 1.0); - _ = Metrics.ComputeProgress().Should().BeApproximately(0.4, 1e-9); - Metrics.FileInflightDone(); + + Metrics.OpStarted(); + Metrics.OpCompleted(); _ = Metrics.ComputeProgress().Should().Be(0.0); } } diff --git a/README.md b/README.md index bce9b80b..2888519b 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ Utility to optimize media files for Direct Play in Plex, Emby, Jellyfin, etc. **Summary:** -- Added always-on runtime metrics with byte-weighted progress, published via `System.Diagnostics.Metrics` and readable with `dotnet-counters`. +- Added always-on runtime metrics with operation-weighted progress, published via `System.Diagnostics.Metrics` and readable with `dotnet-counters`. - Bundled a `counters` wrapper in the Docker image for reading the metrics with a single command. See [Release History](./HISTORY.md) for complete release notes and older versions. @@ -836,7 +836,7 @@ Additional commands for specific tasks, organized by category: ## Runtime Metrics -PlexCleaner publishes always-on runtime metrics via `System.Diagnostics.Metrics` under the `PlexCleaner.Process` meter, so a long-running `process` or `monitor` pass can be watched live with no extra setup. Overall progress is weighted by input bytes, not file count, so a run mixing small and large files reflects the actual work completed. Progress advances smoothly during a long operation because each in-flight file contributes partial credit from the running tool (the full-file scan, the re-encode, and the deinterlace). +PlexCleaner publishes always-on runtime metrics via `System.Diagnostics.Metrics` under the `PlexCleaner.Process` meter, so a long-running `process` or `monitor` pass can be watched live with no extra setup. Overall progress is operation-weighted: each heavy full-file operation on a file (closed-caption and interlace scans, bitrate analysis, re-encode, deinterlace, verify) counts the file's size as work to do when it starts and as work done when it finishes. A run mixing small and large files therefore reflects the actual work completed, and because the per-file path is not fixed the total grows as operations are discovered. The meter publishes `progress.ratio` and `eta.seconds`, the `work.total`/`work.completed` byte counters behind them, `bytes.total` (input size), `files.total`/`files.inflight`/`files.completed`, and per-tool timing histograms. Read the meter with [`dotnet-counters`][dotnet-counters-link]: From f68fa178882e2d9e21a1e19fb4c92de67d38e855 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Sun, 19 Jul 2026 07:38:12 -0700 Subject: [PATCH 2/6] Call the work instruments gauges not counters in the docs work.total and work.completed are run-scoped observable gauges, not Counter instruments, so name them gauges to avoid confusing dotnet-counters and OpenTelemetry consumers. Co-Authored-By: Claude Opus 4.8 --- HISTORY.md | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index 4d3b8961..07b7925d 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -7,7 +7,7 @@ Utility to optimize media files for Direct Play in Plex, Emby, Jellyfin, etc. - Version 3.22: - Added always-on runtime metrics published via `System.Diagnostics.Metrics` under the `PlexCleaner.Process` meter, readable with `dotnet-counters` with no extra infrastructure. - Overall progress is operation-weighted: each heavy full-file operation (the closed-caption and interlace scans, bitrate analysis, re-encode, deinterlace, and verify) counts the file's size as work to do when it starts and as work done when it finishes, so a run mixing tiny and huge files reports the actual work completed, and the total grows as the non-deterministic per-file path is discovered. - - Instruments include the run file total and input byte total, in-flight and active-thread counts, the `work.total`/`work.completed` byte counters and the `progress.ratio` and `eta.seconds` derived from them, cumulative per-outcome counters (completed, modified, errors, verify-failed, and a per-`State`-flag tally), and the `file.duration` and per-tool `tool.duration` histograms. Metrics are aggregate only, with bounded `state` and `tool` tags and no filename tags. + - Instruments include the run file total and input byte total, in-flight and active-thread counts, the `work.total`/`work.completed` byte gauges and the `progress.ratio` and `eta.seconds` derived from them, cumulative per-outcome counters (completed, modified, errors, verify-failed, and a per-`State`-flag tally), and the `file.duration` and per-tool `tool.duration` histograms. Metrics are aggregate only, with bounded `state` and `tool` tags and no filename tags. - The run-scoped gauges reset at the start of every processing pass, so monitor mode and back-to-back commands each report their own run, while the counters stay cumulative for rate display. - Instruments are inert until a listener observes them, so the feature is always on with no configuration flag and negligible idle overhead. - The meter is OpenTelemetry and `dotnet-monitor` compatible, and the Docker image ships a `counters` wrapper, so reading the metrics is `docker exec counters`. diff --git a/README.md b/README.md index 2888519b..6d2f77e9 100644 --- a/README.md +++ b/README.md @@ -836,7 +836,7 @@ Additional commands for specific tasks, organized by category: ## Runtime Metrics -PlexCleaner publishes always-on runtime metrics via `System.Diagnostics.Metrics` under the `PlexCleaner.Process` meter, so a long-running `process` or `monitor` pass can be watched live with no extra setup. Overall progress is operation-weighted: each heavy full-file operation on a file (closed-caption and interlace scans, bitrate analysis, re-encode, deinterlace, verify) counts the file's size as work to do when it starts and as work done when it finishes. A run mixing small and large files therefore reflects the actual work completed, and because the per-file path is not fixed the total grows as operations are discovered. The meter publishes `progress.ratio` and `eta.seconds`, the `work.total`/`work.completed` byte counters behind them, `bytes.total` (input size), `files.total`/`files.inflight`/`files.completed`, and per-tool timing histograms. +PlexCleaner publishes always-on runtime metrics via `System.Diagnostics.Metrics` under the `PlexCleaner.Process` meter, so a long-running `process` or `monitor` pass can be watched live with no extra setup. Overall progress is operation-weighted: each heavy full-file operation on a file (closed-caption and interlace scans, bitrate analysis, re-encode, deinterlace, verify) counts the file's size as work to do when it starts and as work done when it finishes. A run mixing small and large files therefore reflects the actual work completed, and because the per-file path is not fixed the total grows as operations are discovered. The meter publishes `progress.ratio` and `eta.seconds`, the `work.total`/`work.completed` byte gauges behind them, `bytes.total` (input size), `files.total`/`files.inflight`/`files.completed`, and per-tool timing histograms. Read the meter with [`dotnet-counters`][dotnet-counters-link]: From 39ced2799501a6de8ca67cb124bff6590a08ed95 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Sun, 19 Jul 2026 07:46:23 -0700 Subject: [PATCH 3/6] Count operation completion only when the tool actually ran Move OpCompleted after the executed guard so a cancelled or failed-to-start tool is not credited as completed work, which would skew progress and ETA on interrupted runs. A non-zero exit still counts, the tool did the full-file work. Co-Authored-By: Claude Opus 4.8 --- PlexCleaner/FfMpegTool.cs | 16 ++++++++++++---- PlexCleaner/FfProbeTool.cs | 4 ++-- PlexCleaner/HandBrakeTool.cs | 6 +++++- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/PlexCleaner/FfMpegTool.cs b/PlexCleaner/FfMpegTool.cs index 806ae719..50627654 100644 --- a/PlexCleaner/FfMpegTool.cs +++ b/PlexCleaner/FfMpegTool.cs @@ -166,12 +166,12 @@ public VerifyResult VerifyMedia(string fileName) VerifyClassifier.Accumulator classifier = new(); Metrics.OpStarted(); bool executed = ExecuteStreamStdErr(command, classifier.Add, out int exitCode); - Metrics.OpCompleted(); if (!executed) { // Process could not run return VerifyResult.DecodeError; } + Metrics.OpCompleted(); // A non-zero exit is always a failure, fail closed even if stderr shows only the timestamp warning VerifyResult verifyResult = classifier.Result; @@ -352,8 +352,12 @@ string outputName // Execute command Metrics.OpStarted(); bool executed = Execute(command, true, true, out BufferedCommandResult result); + if (!executed) + { + return false; + } Metrics.OpCompleted(); - return executed && (result.ExitCode == 0 || LogFailedResult(result, inputName)); + return result.ExitCode == 0 || LogFailedResult(result, inputName); } public bool ConvertToMkv(string inputName, string outputName) @@ -382,8 +386,12 @@ public bool ConvertToMkv(string inputName, string outputName) // Execute command Metrics.OpStarted(); bool executed = Execute(command, true, true, out BufferedCommandResult result); + if (!executed) + { + return false; + } Metrics.OpCompleted(); - return executed && (result.ExitCode == 0 || LogFailedResult(result, inputName)); + return result.ExitCode == 0 || LogFailedResult(result, inputName); } public bool SetTimestamps(string inputName, string outputName) @@ -512,11 +520,11 @@ public bool GetIdetText(string fileName, out string text) // Execute command Metrics.OpStarted(); bool executed = Execute(command, true, true, out BufferedCommandResult result); - Metrics.OpCompleted(); if (!executed) { return false; } + Metrics.OpCompleted(); text = result.StandardError.Trim(); return result.ExitCode == 0 || LogFailedResult(result, fileName); } diff --git a/PlexCleaner/FfProbeTool.cs b/PlexCleaner/FfProbeTool.cs index 791a835f..6211a896 100644 --- a/PlexCleaner/FfProbeTool.cs +++ b/PlexCleaner/FfProbeTool.cs @@ -228,11 +228,11 @@ public bool GetClosedCaptions(string fileName, out bool hasClosedCaptions) // Execute command Metrics.OpStarted(); bool executed = Execute(command, false, true, out BufferedCommandResult result); - Metrics.OpCompleted(); if (!executed) { return false; } + Metrics.OpCompleted(); if (result.ExitCode != 0) { return LogFailedResult(result, fileName); @@ -318,13 +318,13 @@ bool quickScan // Get packet list Metrics.OpStarted(); bool got = GetPackets(command, packetFunc, out string error); - Metrics.OpCompleted(); if (!got) { Log.Error("Failed to get analysis packets : {FileName}", fileName); LogErrorOutput(error); return false; } + Metrics.OpCompleted(); return true; } diff --git a/PlexCleaner/HandBrakeTool.cs b/PlexCleaner/HandBrakeTool.cs index edd57558..26256a48 100644 --- a/PlexCleaner/HandBrakeTool.cs +++ b/PlexCleaner/HandBrakeTool.cs @@ -142,8 +142,12 @@ bool deInterlace // Execute command Metrics.OpStarted(); bool executed = Execute(command, true, true, out BufferedCommandResult result); + if (!executed) + { + return false; + } Metrics.OpCompleted(); - return executed && (result.ExitCode == 0 || LogFailedResult(result, inputName)); + return result.ExitCode == 0 || LogFailedResult(result, inputName); } [GeneratedRegex( From 49f837e17d4454d06dc5103596455f0209f368d4 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Sun, 19 Jul 2026 07:53:34 -0700 Subject: [PATCH 4/6] Count the packet scan when it ran but exited non-zero GetPackets returns false on a non-zero exit where ffprobe still ran the scan, so gate OpCompleted on cancellation instead of the return value, otherwise a ran-but-non-zero scan strands its work in the total and skews progress for the rest of the run. Co-Authored-By: Claude Opus 4.8 --- PlexCleaner/FfProbeTool.cs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/PlexCleaner/FfProbeTool.cs b/PlexCleaner/FfProbeTool.cs index 6211a896..8aac7450 100644 --- a/PlexCleaner/FfProbeTool.cs +++ b/PlexCleaner/FfProbeTool.cs @@ -318,13 +318,17 @@ bool quickScan // Get packet list Metrics.OpStarted(); bool got = GetPackets(command, packetFunc, out string error); + // GetPackets also returns false on a non-zero exit where the scan still ran, so count completion unless cancelled + if (!Program.IsCancelled()) + { + Metrics.OpCompleted(); + } if (!got) { Log.Error("Failed to get analysis packets : {FileName}", fileName); LogErrorOutput(error); return false; } - Metrics.OpCompleted(); return true; } From 59372f063649e6e2156f04d76c3268d8ca0ee0b6 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Sun, 19 Jul 2026 07:59:32 -0700 Subject: [PATCH 5/6] Credit the packet scan only when ffprobe actually ran Gate OpCompleted on the scan having run, evidenced by exit 0 or captured stderr, instead of a cancellation check, so a cancellation or a failure to start (empty error) is excluded while a ran-but-non-zero scan is still counted. Co-Authored-By: Claude Opus 4.8 --- PlexCleaner/FfProbeTool.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/PlexCleaner/FfProbeTool.cs b/PlexCleaner/FfProbeTool.cs index 8aac7450..f21b8bba 100644 --- a/PlexCleaner/FfProbeTool.cs +++ b/PlexCleaner/FfProbeTool.cs @@ -318,8 +318,8 @@ bool quickScan // Get packet list Metrics.OpStarted(); bool got = GetPackets(command, packetFunc, out string error); - // GetPackets also returns false on a non-zero exit where the scan still ran, so count completion unless cancelled - if (!Program.IsCancelled()) + // GetPackets is false on a non-zero exit where the scan still ran, count completion when it ran (exit 0 or stderr output), not on cancellation or a failure to start + if (got || !string.IsNullOrEmpty(error)) { Metrics.OpCompleted(); } From 96f1ea0e62b9f732faddeaf117576cf6cded4612 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Sun, 19 Jul 2026 08:07:28 -0700 Subject: [PATCH 6/6] Roll back discovered work when an operation never runs OpStarted adds the file size to the work total up front, so an operation that is cancelled or fails to start would leave the total permanently inflated and stop progress converging. Add OpAborted to roll that size back out, and call it on every abort path so each started operation resolves to exactly one of completed or aborted. Co-Authored-By: Claude Opus 4.8 --- PlexCleaner/FfMpegTool.cs | 4 ++++ PlexCleaner/FfProbeTool.cs | 5 +++++ PlexCleaner/HandBrakeTool.cs | 1 + PlexCleaner/Metrics.cs | 4 ++++ PlexCleanerTests/MetricsTests.cs | 20 ++++++++++++++++++++ 5 files changed, 34 insertions(+) diff --git a/PlexCleaner/FfMpegTool.cs b/PlexCleaner/FfMpegTool.cs index 50627654..52f51044 100644 --- a/PlexCleaner/FfMpegTool.cs +++ b/PlexCleaner/FfMpegTool.cs @@ -169,6 +169,7 @@ public VerifyResult VerifyMedia(string fileName) if (!executed) { // Process could not run + Metrics.OpAborted(); return VerifyResult.DecodeError; } Metrics.OpCompleted(); @@ -354,6 +355,7 @@ string outputName bool executed = Execute(command, true, true, out BufferedCommandResult result); if (!executed) { + Metrics.OpAborted(); return false; } Metrics.OpCompleted(); @@ -388,6 +390,7 @@ public bool ConvertToMkv(string inputName, string outputName) bool executed = Execute(command, true, true, out BufferedCommandResult result); if (!executed) { + Metrics.OpAborted(); return false; } Metrics.OpCompleted(); @@ -522,6 +525,7 @@ public bool GetIdetText(string fileName, out string text) bool executed = Execute(command, true, true, out BufferedCommandResult result); if (!executed) { + Metrics.OpAborted(); return false; } Metrics.OpCompleted(); diff --git a/PlexCleaner/FfProbeTool.cs b/PlexCleaner/FfProbeTool.cs index f21b8bba..9ac959bd 100644 --- a/PlexCleaner/FfProbeTool.cs +++ b/PlexCleaner/FfProbeTool.cs @@ -230,6 +230,7 @@ public bool GetClosedCaptions(string fileName, out bool hasClosedCaptions) bool executed = Execute(command, false, true, out BufferedCommandResult result); if (!executed) { + Metrics.OpAborted(); return false; } Metrics.OpCompleted(); @@ -323,6 +324,10 @@ bool quickScan { Metrics.OpCompleted(); } + else + { + Metrics.OpAborted(); + } if (!got) { Log.Error("Failed to get analysis packets : {FileName}", fileName); diff --git a/PlexCleaner/HandBrakeTool.cs b/PlexCleaner/HandBrakeTool.cs index 26256a48..90acd4a1 100644 --- a/PlexCleaner/HandBrakeTool.cs +++ b/PlexCleaner/HandBrakeTool.cs @@ -144,6 +144,7 @@ bool deInterlace bool executed = Execute(command, true, true, out BufferedCommandResult result); if (!executed) { + Metrics.OpAborted(); return false; } Metrics.OpCompleted(); diff --git a/PlexCleaner/Metrics.cs b/PlexCleaner/Metrics.cs index 67872c7f..409ab0c1 100644 --- a/PlexCleaner/Metrics.cs +++ b/PlexCleaner/Metrics.cs @@ -154,6 +154,10 @@ internal static void OpStarted() => internal static void OpCompleted() => Interlocked.Add(ref s_runWorkCompleted, s_currentFileSize.Value); + // The heavy operation never ran, roll its size back out of the total so progress can still converge. + internal static void OpAborted() => + Interlocked.Add(ref s_runWorkTotal, -s_currentFileSize.Value); + internal static void FileCompleted(TimeSpan wall) { s_filesCompleted.Add(1); diff --git a/PlexCleanerTests/MetricsTests.cs b/PlexCleanerTests/MetricsTests.cs index fbd38d33..3e74231a 100644 --- a/PlexCleanerTests/MetricsTests.cs +++ b/PlexCleanerTests/MetricsTests.cs @@ -157,6 +157,26 @@ public void ComputeProgress_IsCompletedOverDiscoveredWork() Metrics.FileInflightDone(); } + [Fact] + public void OpAborted_RollsBackTheStartedWork() + { + // An operation that never ran rolls its size back out of the total so progress still converges + Metrics.BeginRun(1, 1000); + Metrics.FileStarted(400); + + Metrics.OpStarted(); + Metrics.OpCompleted(); + _ = Metrics.ComputeProgress().Should().Be(1.0); + + // A second operation starts then aborts, the total returns to the completed work + Metrics.OpStarted(); + _ = Metrics.ComputeProgress().Should().BeApproximately(0.5, 1e-9); + Metrics.OpAborted(); + _ = Metrics.ComputeProgress().Should().Be(1.0); + + Metrics.FileInflightDone(); + } + [Fact] public void OpAfterInflightDone_CountsNothing() {