From b8912ea148d1f14f2e547f27550bacc51e849ec6 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Sat, 18 Jul 2026 12:48:02 -0700 Subject: [PATCH 1/6] Publish runtime metrics and byte-weighted progress via Metrics Add an always-on System.Diagnostics.Metrics meter (PlexCleaner.Process) so a long-running process or monitor pass can be inspected live with dotnet-counters, no extra infrastructure. Overall progress is weighted by input bytes, not file count, so a run mixing tiny and huge files reports actual work done. - Metrics.cs owns the meter, the instruments, and Interlocked run-scoped state; observable-gauge callbacks only read, so the parallel loop needs no lock. - ProcessDriver.ProcessFiles drives files/bytes/in-flight, byte-weighted progress.ratio and eta.seconds, and file.duration; it sums input sizes once up front and credits the same size at completion. Run-scoped gauges reset per pass; counters stay cumulative for rate display. - Process.ProcessFiles records per-SidecarFile.StatesType outcomes; MediaTool and the ffprobe packet reader record tool.duration. Tags are bounded (state, tool) with no filename tags. - v1 gives in-flight files no partial credit until they finish; per-file child-process progress is a separate follow-up. - Docker ships a "counters" wrapper so reading the meter is "docker exec counters". Zero new packages (Metrics is in the BCL). Implements #848 v1. Co-Authored-By: Claude Opus 4.8 --- ARCHITECTURE.md | 8 ++ Docker/Dockerfile | 3 + Docker/counters.sh | 17 +++ HISTORY.md | 7 ++ PlexCleaner/FfProbeTool.cs | 9 ++ PlexCleaner/MediaTool.cs | 17 +++ PlexCleaner/Metrics.cs | 180 +++++++++++++++++++++++++++++++ PlexCleaner/Process.cs | 11 ++ PlexCleaner/ProcessDriver.cs | 44 +++++++- PlexCleaner/Program.cs | 1 + PlexCleanerTests/MetricsTests.cs | 138 ++++++++++++++++++++++++ README.md | 10 ++ version.json | 2 +- 13 files changed, 444 insertions(+), 3 deletions(-) create mode 100644 Docker/counters.sh create mode 100644 PlexCleaner/Metrics.cs create mode 100644 PlexCleanerTests/MetricsTests.cs diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 49451373..ae428f09 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -75,6 +75,14 @@ All external process execution uses [CliWrap](https://github.com/Tyrrrz/CliWrap) - See `MediaTool.cs` for base execution patterns - All tool execution supports cancellation via `Program.CancelToken()` +### Runtime Metrics + +`Metrics.cs` owns a single `System.Diagnostics.Metrics.Meter` (`PlexCleaner.Process`) published for the whole process and read externally with `dotnet-counters` (no config flag; instruments are inert until observed). + +- Hooks: `ProcessDriver.ProcessFiles` (the choke point every command and monitor cycle funnels through) drives the file/byte/in-flight instruments and the byte-weighted `progress.ratio`; `Process.ProcessFiles` records the per-`SidecarFile.StatesType` outcomes; `MediaTool` execution paths record `tool.duration`. +- Progress is weighted by input bytes (summed once up front and credited at completion from the same map), not file count. +- Run-scoped gauges (totals, in-flight, progress, ETA) reset per `ProcessFiles` call; the counters stay cumulative for the process. All writers use `Interlocked`, so the parallel loop needs no lock; observable-gauge callbacks only read. Tags are bounded (`state`, `tool`) — no filename tags. + ### Sidecar File System Critical performance feature - DO NOT break compatibility: diff --git a/Docker/Dockerfile b/Docker/Dockerfile index c37a6ed6..0ecd83cd 100644 --- a/Docker/Dockerfile +++ b/Docker/Dockerfile @@ -134,6 +134,9 @@ COPY --chmod=ug=rwx,o=rx ./Docker/InstallDebugTools.sh ./ RUN ./InstallDebugTools.sh \ && rm -rf ./InstallDebugTools.sh +# Wrapper to read the PlexCleaner.Process metrics: "docker exec counters" +COPY --chmod=ug=rwx,o=rx ./Docker/counters.sh /usr/local/bin/counters + # Copy version script COPY --chmod=ug=rwx,o=rx ./Docker/Version.sh /PlexCleaner/ diff --git a/Docker/counters.sh b/Docker/counters.sh new file mode 100644 index 00000000..56261ee2 --- /dev/null +++ b/Docker/counters.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +# Run dotnet-counters against the in-container PlexCleaner process. +# The single-file tool needs a writable extract dir, and the app is PID 1 in the container. +# Usage from the host: +# docker exec counters # live monitor of the PlexCleaner.Process meter +# docker exec counters collect ... # any other dotnet-counters verb/args, passed through + +set -euo pipefail + +export DOTNET_BUNDLE_EXTRACT_BASE_DIR="${DOTNET_BUNDLE_EXTRACT_BASE_DIR:-/tmp}" + +if [[ $# -eq 0 ]]; then + set -- monitor -p 1 --counters PlexCleaner.Process +fi + +exec /dotnet-tools/dotnet-counters "$@" diff --git a/HISTORY.md b/HISTORY.md index 2d4f954a..e2d24b5e 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -4,6 +4,13 @@ Utility to optimize media files for Direct Play in Plex, Emby, Jellyfin, etc. ## Release History +- Version 3.22: + - Added always-on runtime metrics published via `System.Diagnostics.Metrics` under the `PlexCleaner.Process` meter, readable with `dotnet-counters` with no extra infrastructure ([#848](https://github.com/ptr727/PlexCleaner/issues/848)). + - Overall progress is weighted by input bytes rather than file count, so a run mixing tiny and huge files reports the actual work completed; in this first version an in-flight file contributes no partial credit until it finishes. + - Instruments include the run file and byte totals, in-flight and active-thread counts, completed bytes, the weighted `progress.ratio` and `eta.seconds`, cumulative per-outcome counters (completed, modified, errors, verify-failed, and a per-`State`-flag tally), and the `file.duration` and per-tool `tool.duration` histograms. Metrics are aggregate only, with bounded `state` and `tool` tags and no filename tags. + - The run-scoped gauges reset at the start of every processing pass, so monitor mode and back-to-back commands each report their own run, while the counters stay cumulative for rate display. + - Instruments are inert until a listener observes them, so the feature is always on with no configuration flag and negligible idle overhead. + - The Docker image ships a `counters` wrapper, so reading the metrics is `docker exec counters`. - Version 3.21: - Repair non-monotonic DTS muxer warnings losslessly instead of failing repair permanently. - `ffmpeg -f null` can exit `0` yet emit `Application provided invalid, non monotonically increasing dts to muxer` for files that may decode and play correctly. diff --git a/PlexCleaner/FfProbeTool.cs b/PlexCleaner/FfProbeTool.cs index c30a59b8..0fb1c12d 100644 --- a/PlexCleaner/FfProbeTool.cs +++ b/PlexCleaner/FfProbeTool.cs @@ -1,3 +1,4 @@ +using System.Diagnostics; using System.Runtime.CompilerServices; using System.Text; using System.Text.Json; @@ -79,6 +80,7 @@ public bool GetPackets( ) { int processId = -1; + long startTimestamp = Stopwatch.GetTimestamp(); try { // Pipe target to deserialize JSON packets @@ -194,6 +196,13 @@ public bool GetPackets( { return (false, string.Empty); } + finally + { + Metrics.RecordToolDuration( + GetToolType(), + Stopwatch.GetElapsedTime(startTimestamp).TotalMilliseconds + ); + } } public bool GetClosedCaptions(string fileName, out bool hasClosedCaptions) diff --git a/PlexCleaner/MediaTool.cs b/PlexCleaner/MediaTool.cs index 0eb9c7a3..af877c81 100644 --- a/PlexCleaner/MediaTool.cs +++ b/PlexCleaner/MediaTool.cs @@ -1,3 +1,4 @@ +using System.Diagnostics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Text; @@ -173,6 +174,7 @@ public bool Execute( { bufferedCommandResult = null!; int processId = -1; + long startTimestamp = Stopwatch.GetTimestamp(); try { StringBuilder stdOutBuilder = new(); @@ -223,6 +225,13 @@ public bool Execute( { return false; } + finally + { + Metrics.RecordToolDuration( + GetToolType(), + Stopwatch.GetElapsedTime(startTimestamp).TotalMilliseconds + ); + } } public bool ExecuteStreamStdErr( @@ -234,6 +243,7 @@ public bool ExecuteStreamStdErr( { exitCode = -1; int processId = -1; + long startTimestamp = Stopwatch.GetTimestamp(); try { // Stream stderr line by line to the caller instead of buffering it @@ -285,6 +295,13 @@ public bool ExecuteStreamStdErr( { return false; } + finally + { + Metrics.RecordToolDuration( + GetToolType(), + Stopwatch.GetElapsedTime(startTimestamp).TotalMilliseconds + ); + } } public static PipeTarget ToStringBuilder(StringBuilder stringBuilder) => diff --git a/PlexCleaner/Metrics.cs b/PlexCleaner/Metrics.cs new file mode 100644 index 00000000..0231cd54 --- /dev/null +++ b/PlexCleaner/Metrics.cs @@ -0,0 +1,180 @@ +using System.Diagnostics; +using System.Diagnostics.Metrics; + +namespace PlexCleaner; + +// Always-on runtime metrics published via System.Diagnostics.Metrics, readable with dotnet-counters +// (no config flag; instruments are inert until a listener observes them). Overall progress is +// weighted by input bytes, not file count, so a run with one small and one huge file reports real +// work done. Aggregate only: no filename tags (unbounded cardinality) - the state and tool tags are +// bounded enums. +internal static class Metrics +{ + private static readonly Meter s_meter = new("PlexCleaner.Process"); + + // Cumulative counters live for the whole process (correct for dotnet-counters rate display) and + // are NOT reset between runs or monitor cycles. + private static readonly Counter s_filesCompleted = s_meter.CreateCounter( + "plexcleaner.files.completed", + description: "Files finished, any outcome" + ); + private static readonly Counter s_filesModified = s_meter.CreateCounter( + "plexcleaner.files.modified", + description: "Files whose media was changed" + ); + private static readonly Counter s_filesErrors = s_meter.CreateCounter( + "plexcleaner.files.errors", + description: "Files that errored" + ); + private static readonly Counter s_filesVerifyFailed = s_meter.CreateCounter( + "plexcleaner.files.verifyfailed", + description: "Files that failed verification" + ); + private static readonly Counter s_filesProcessed = s_meter.CreateCounter( + "plexcleaner.files.processed", + description: "Per-outcome tally, tagged by each State flag set" + ); + + private static readonly Histogram s_fileDuration = s_meter.CreateHistogram( + "plexcleaner.file.duration", + unit: "ms", + description: "Per-file wall-clock time" + ); + private static readonly Histogram s_toolDuration = s_meter.CreateHistogram( + "plexcleaner.tool.duration", + unit: "ms", + description: "Per media-tool invocation time, tagged by tool" + ); + + // Run-scoped state, reset by BeginRun and only read by the observable gauges. All access is via + // Interlocked so the parallel per-file loop needs no lock. + private static long s_runFilesTotal; + private static long s_runBytesTotal; + private static long s_runBytesCompleted; + private static long s_runInflight; + private static long s_runStartTimestamp; + + static Metrics() + { + _ = s_meter.CreateObservableGauge( + "plexcleaner.files.total", + () => Interlocked.Read(ref s_runFilesTotal), + description: "Files in the current run" + ); + _ = s_meter.CreateObservableGauge( + "plexcleaner.files.inflight", + () => Interlocked.Read(ref s_runInflight), + description: "Files currently processing" + ); + _ = s_meter.CreateObservableGauge( + "plexcleaner.threads.active", + () => (long)(Program.Options?.ThreadCount ?? 0), + description: "Configured worker threads" + ); + _ = s_meter.CreateObservableGauge( + "plexcleaner.bytes.total", + () => Interlocked.Read(ref s_runBytesTotal), + unit: "By", + description: "Sum of input sizes in the current run" + ); + _ = s_meter.CreateObservableGauge( + "plexcleaner.bytes.completed", + () => Interlocked.Read(ref s_runBytesCompleted), + unit: "By", + description: "Bytes of finished files (no partial credit for in-flight files)" + ); + _ = s_meter.CreateObservableGauge( + "plexcleaner.progress.ratio", + ComputeProgress, + description: "Byte-weighted overall progress [0..1]" + ); + _ = s_meter.CreateObservableGauge( + "plexcleaner.eta.seconds", + ComputeEtaSeconds, + unit: "s", + description: "Estimated time remaining" + ); + } + + // Start a new run: reset the run-scoped gauges and the ETA clock. Called at every + // ProcessDriver.ProcessFiles entry, so back-to-back commands and each monitor cycle restart clean. + internal static void BeginRun(long totalFiles, long totalBytes) + { + _ = Interlocked.Exchange(ref s_runFilesTotal, totalFiles); + _ = Interlocked.Exchange(ref s_runBytesTotal, totalBytes); + _ = Interlocked.Exchange(ref s_runBytesCompleted, 0); + _ = Interlocked.Exchange(ref s_runInflight, 0); + _ = Interlocked.Exchange(ref s_runStartTimestamp, Stopwatch.GetTimestamp()); + } + + internal static void FileStarted() => Interlocked.Increment(ref s_runInflight); + + internal static void FileInflightDone() => Interlocked.Decrement(ref s_runInflight); + + // A file finished (any outcome): credit its whole size (no partial credit in v1) and record its + // wall-clock time. + internal static void FileCompleted(long sizeBytes, TimeSpan wall) + { + _ = Interlocked.Add(ref s_runBytesCompleted, sizeBytes); + s_filesCompleted.Add(1); + s_fileDuration.Record(wall.TotalMilliseconds); + } + + internal static void FileErrored() => s_filesErrors.Add(1); + + internal static void RecordModified() => s_filesModified.Add(1); + + internal static void RecordVerifyFailed() => s_filesVerifyFailed.Add(1); + + internal static void RecordStates(SidecarFile.StatesType state) + { + foreach (SidecarFile.StatesType flag in EnumerateSetStates(state)) + { + s_filesProcessed.Add(1, new KeyValuePair("state", flag.ToString())); + } + } + + internal static void RecordToolDuration(MediaTool.ToolType tool, double milliseconds) => + s_toolDuration.Record( + milliseconds, + new KeyValuePair("tool", tool.ToString()) + ); + + internal static void Dispose() => s_meter.Dispose(); + + // Byte-weighted progress; guards a zero (or not-yet-started) total. + internal static double ComputeProgress() + { + long total = Interlocked.Read(ref s_runBytesTotal); + return total <= 0 ? 0.0 : (double)Interlocked.Read(ref s_runBytesCompleted) / total; + } + + // Linear extrapolation from weighted progress and elapsed time; 0 until there is progress, and + // never a non-finite value (which dotnet-counters would render as NaN/Infinity). + internal static double ComputeEtaSeconds() + { + double ratio = ComputeProgress(); + if (ratio <= 0.0) + { + return 0.0; + } + double elapsed = Stopwatch + .GetElapsedTime(Interlocked.Read(ref s_runStartTimestamp)) + .TotalSeconds; + double eta = elapsed * (1.0 - ratio) / ratio; + return double.IsFinite(eta) ? eta : 0.0; + } + + internal static IEnumerable EnumerateSetStates( + SidecarFile.StatesType state + ) + { + foreach (SidecarFile.StatesType flag in Enum.GetValues()) + { + if (flag != SidecarFile.StatesType.None && state.HasFlag(flag)) + { + yield return flag; + } + } + } +} diff --git a/PlexCleaner/Process.cs b/PlexCleaner/Process.cs index e35919a7..daea4fea 100644 --- a/PlexCleaner/Process.cs +++ b/PlexCleaner/Process.cs @@ -506,6 +506,17 @@ out string? failedOperation return processResult; } + // Per-outcome metrics: this is the only place the State flags are known + Metrics.RecordStates(state); + if (modified) + { + Metrics.RecordModified(); + } + if (state.HasFlag(SidecarFile.StatesType.VerifyFailed)) + { + Metrics.RecordVerifyFailed(); + } + // Save result lock (resultLock) { diff --git a/PlexCleaner/ProcessDriver.cs b/PlexCleaner/ProcessDriver.cs index 24c0f6fe..3ffdece4 100644 --- a/PlexCleaner/ProcessDriver.cs +++ b/PlexCleaner/ProcessDriver.cs @@ -117,6 +117,28 @@ Func taskFunc // Process all files in parallel int totalCount = fileList.Count; + + // Size map for the byte-weighted progress metrics: sum once up front and credit the same size + // at completion, so a remux or rename mid-run cannot drift the total. Missing files weight as + // zero. BeginRun resets the run-scoped gauges, covering back-to-back commands and monitor cycles. + Dictionary fileSizes = new(totalCount, StringComparer.Ordinal); + long totalBytes = 0; + foreach (string file in fileList) + { + long length = 0; + try + { + length = new FileInfo(file).Length; + } + catch (Exception e) when (Log.Logger.LogAndHandle(e)) + { + // Length unavailable: weight this file as zero + } + fileSizes[file] = length; + totalBytes += length; + } + Metrics.BeginRun(totalCount, totalBytes); + int processedCount = 0; int errorCount = 0; bool error = false; @@ -167,6 +189,11 @@ Func taskFunc processedPercentage, fileName ); + // Credit skipped files so byte-weighted progress can still reach 1.0 + Metrics.FileCompleted( + fileSizes.GetValueOrDefault(fileName), + TimeSpan.Zero + ); continue; } @@ -177,9 +204,20 @@ Func taskFunc fileName ); - // Perform the task, timing this file's work + // Perform the task, timing this file's work. Track the in-flight count around + // the task in a finally so a cancellation cannot leak it. + long fileSize = fileSizes.GetValueOrDefault(fileName); + Metrics.FileStarted(); long startTimestamp = Stopwatch.GetTimestamp(); - bool taskResult = taskFunc(fileName); + bool taskResult; + try + { + taskResult = taskFunc(fileName); + } + finally + { + Metrics.FileInflightDone(); + } TimeSpan taskElapsed = Stopwatch.GetElapsedTime(startTimestamp); // Handle cancel request @@ -191,6 +229,7 @@ Func taskFunc // Error Log.Error("{TaskName} Error : {FileName}", taskName, fileName); _ = Interlocked.Increment(ref errorCount); + Metrics.FileErrored(); } // Log completion % after task completes @@ -198,6 +237,7 @@ Func taskFunc Interlocked.Increment(ref processedCount), totalCount ); + Metrics.FileCompleted(fileSize, taskElapsed); Log.Information( "{TaskName} ({Processed:F2}%) Elapsed : {Elapsed:l} : After : {FileName}", taskName, diff --git a/PlexCleaner/Program.cs b/PlexCleaner/Program.cs index e22b31fa..298f2318 100644 --- a/PlexCleaner/Program.cs +++ b/PlexCleaner/Program.cs @@ -114,6 +114,7 @@ private static int Main(string[] args) Log.Logger.LogOverrideContext().Information("Exit Code : {ExitCode}", exitCode); Log.CloseAndFlush(); s_libraryLoggerFactory?.Dispose(); + Metrics.Dispose(); return exitCode; } diff --git a/PlexCleanerTests/MetricsTests.cs b/PlexCleanerTests/MetricsTests.cs new file mode 100644 index 00000000..6187fae5 --- /dev/null +++ b/PlexCleanerTests/MetricsTests.cs @@ -0,0 +1,138 @@ +using System.Diagnostics.Metrics; +using AwesomeAssertions; +using PlexCleaner; +using Xunit; + +namespace PlexCleanerTests; + +// Metrics is process-static, so run these in the non-parallel collection and reset run state with +// BeginRun at the start of every test. +[Collection("Sequential")] +public class MetricsTests +{ + [Fact] + public void ComputeProgress_IsByteWeightedNotCountWeighted() + { + // Two files totalling 1000 bytes; completing only the 900-byte one is 90% of the work, not + // 50% of the file count. + Metrics.BeginRun(2, 1000); + Metrics.FileCompleted(900, TimeSpan.Zero); + + _ = Metrics.ComputeProgress().Should().BeApproximately(0.9, 1e-9); + } + + [Fact] + public void ComputeProgress_ZeroTotal_IsZero() + { + Metrics.BeginRun(0, 0); + + _ = Metrics.ComputeProgress().Should().Be(0.0); + } + + [Fact] + public void ComputeProgress_AllBytesDone_IsOne() + { + Metrics.BeginRun(1, 500); + Metrics.FileCompleted(500, TimeSpan.Zero); + + _ = Metrics.ComputeProgress().Should().Be(1.0); + } + + [Fact] + public void ComputeEtaSeconds_NoProgress_IsZero() + { + Metrics.BeginRun(2, 1000); + + _ = Metrics.ComputeEtaSeconds().Should().Be(0.0); + } + + [Fact] + public void ComputeEtaSeconds_PartialProgress_IsFiniteAndNonNegative() + { + Metrics.BeginRun(2, 1000); + Metrics.FileCompleted(400, TimeSpan.Zero); + + double eta = Metrics.ComputeEtaSeconds(); + + _ = double.IsFinite(eta).Should().BeTrue(); + _ = eta.Should().BeGreaterThanOrEqualTo(0.0); + } + + [Fact] + public void EnumerateSetStates_ReturnsEachSetFlag() + { + SidecarFile.StatesType state = + SidecarFile.StatesType.ReMuxed + | SidecarFile.StatesType.Verified + | SidecarFile.StatesType.ClearedTags; + + List flags = [.. Metrics.EnumerateSetStates(state)]; + + _ = flags + .Should() + .BeEquivalentTo([ + SidecarFile.StatesType.ReMuxed, + SidecarFile.StatesType.Verified, + SidecarFile.StatesType.ClearedTags, + ]); + } + + [Fact] + public void EnumerateSetStates_None_IsEmpty() => + _ = Metrics.EnumerateSetStates(SidecarFile.StatesType.None).Should().BeEmpty(); + + [Fact] + public void Instruments_AreObservableViaMeterListener() + { + List<(string Name, long Value, KeyValuePair[] Tags)> longs = []; + List<(string Name, double Value)> doubles = []; + + using MeterListener listener = new(); + listener.InstrumentPublished = (instrument, meterListener) => + { + if (instrument.Meter.Name == "PlexCleaner.Process") + { + meterListener.EnableMeasurementEvents(instrument); + } + }; + listener.SetMeasurementEventCallback( + (instrument, measurement, tags, _) => + longs.Add((instrument.Name, measurement, tags.ToArray())) + ); + listener.SetMeasurementEventCallback( + (instrument, measurement, _, _) => doubles.Add((instrument.Name, measurement)) + ); + listener.Start(); + + // Two files start; one finishes 400 of 1000 bytes (leaves flight and is credited); record a + // two-flag outcome for it + Metrics.BeginRun(2, 1000); + Metrics.FileStarted(); + Metrics.FileStarted(); + Metrics.FileInflightDone(); + Metrics.FileCompleted(400, TimeSpan.Zero); + Metrics.RecordStates(SidecarFile.StatesType.ReMuxed | SidecarFile.StatesType.Verified); + listener.RecordObservableInstruments(); + + // The counter fired one measurement per set flag with the state tag + List states = + [ + .. longs + .Where(m => m.Name == "plexcleaner.files.processed") + .Select(m => m.Tags.Single(t => t.Key == "state").Value?.ToString()), + ]; + _ = states.Should().BeEquivalentTo(["ReMuxed", "Verified"]); + + // One of two started files is still in flight, and progress is byte-weighted + _ = longs + .Should() + .ContainSingle(m => m.Name == "plexcleaner.files.inflight") + .Which.Value.Should() + .Be(1); + _ = doubles + .Should() + .ContainSingle(m => m.Name == "plexcleaner.progress.ratio") + .Which.Value.Should() + .BeApproximately(0.4, 1e-9); + } +} diff --git a/README.md b/README.md index 159c6e78..983cf224 100644 --- a/README.md +++ b/README.md @@ -86,6 +86,7 @@ See [Installation](#installation) for detailed setup instructions and other plat - [Process Command](#process-command) - [Monitor Command](#monitor-command) - [Other Commands](#other-commands) +- [Runtime Metrics](#runtime-metrics) - [Custom Plugins](#custom-plugins) - [Testing](#testing) - [Unit Testing](#unit-testing) @@ -834,6 +835,15 @@ Additional commands for specific tasks, organized by category: - `getmediainfo`: - Print media file information and track details. +## Runtime Metrics + +PlexCleaner publishes always-on runtime metrics via `System.Diagnostics.Metrics` under the `PlexCleaner.Process` meter, so a long-running `process` or `monitor` pass can be watched live with no extra setup. Overall progress is weighted by input bytes, not file count, so a run mixing small and large files reflects the actual work completed. + +Read the meter with [`dotnet-counters`](https://learn.microsoft.com/dotnet/core/diagnostics/dotnet-counters): + +- Local: `dotnet-counters monitor -p PlexCleaner.Process` +- Docker: `docker exec counters` (a bundled wrapper that targets the in-container process) + ## Custom Plugins The `custom` command runs a user-provided plugin assembly over the media files, reusing PlexCleaner's file iteration and processing. This is useful for bespoke, targeted re-processing that the built-in commands do not cover, for example re-running a newly added or fixed verification check on a library where files are already marked as verified, without paying for a full re-verification. diff --git a/version.json b/version.json index 8c723482..329e7634 100644 --- a/version.json +++ b/version.json @@ -1,6 +1,6 @@ { "$schema": "https://raw.githubusercontent.com/dotnet/Nerdbank.GitVersioning/master/src/NerdBank.GitVersioning/version.schema.json", - "version": "3.21", + "version": "3.22", "publicReleaseRefSpec": [ "^refs/heads/main$" ], From 355c7677af3d5d9a658758b30693d1e68e71ef38 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Sat, 18 Jul 2026 12:57:48 -0700 Subject: [PATCH 2/6] Address review: scope metrics to processed files, ASCII, consistent counters cmd - ProcessDriver bases the byte-size map and BeginRun totals on the files that will actually be processed (filters to MKV when mkvFilesOnly), and no longer credits skipped non-MKV files as completed, so progress/ETA and files.total reflect real work for the MKV-only commands. - README local dotnet-counters example uses --counters, matching the wrapper. - Replace an em dash with a hyphen in ARCHITECTURE.md (ASCII-only guidance). Co-Authored-By: Claude Opus 4.8 --- ARCHITECTURE.md | 2 +- PlexCleaner/ProcessDriver.cs | 13 +++++++------ README.md | 4 ++-- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index ae428f09..32fb7409 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -81,7 +81,7 @@ All external process execution uses [CliWrap](https://github.com/Tyrrrz/CliWrap) - Hooks: `ProcessDriver.ProcessFiles` (the choke point every command and monitor cycle funnels through) drives the file/byte/in-flight instruments and the byte-weighted `progress.ratio`; `Process.ProcessFiles` records the per-`SidecarFile.StatesType` outcomes; `MediaTool` execution paths record `tool.duration`. - Progress is weighted by input bytes (summed once up front and credited at completion from the same map), not file count. -- Run-scoped gauges (totals, in-flight, progress, ETA) reset per `ProcessFiles` call; the counters stay cumulative for the process. All writers use `Interlocked`, so the parallel loop needs no lock; observable-gauge callbacks only read. Tags are bounded (`state`, `tool`) — no filename tags. +- Run-scoped gauges (totals, in-flight, progress, ETA) reset per `ProcessFiles` call; the counters stay cumulative for the process. All writers use `Interlocked`, so the parallel loop needs no lock; observable-gauge callbacks only read. Tags are bounded (`state`, `tool`) - no filename tags. ### Sidecar File System diff --git a/PlexCleaner/ProcessDriver.cs b/PlexCleaner/ProcessDriver.cs index 3ffdece4..ef74a1a9 100644 --- a/PlexCleaner/ProcessDriver.cs +++ b/PlexCleaner/ProcessDriver.cs @@ -125,6 +125,12 @@ Func taskFunc long totalBytes = 0; foreach (string file in fileList) { + // Weight only the files that will actually be processed: a mkvFilesOnly command skips + // non-MKV files, so they are excluded from the totals and never credited as completed. + if (mkvFilesOnly && !SidecarFile.IsMkvFile(file)) + { + continue; + } long length = 0; try { @@ -137,7 +143,7 @@ Func taskFunc fileSizes[file] = length; totalBytes += length; } - Metrics.BeginRun(totalCount, totalBytes); + Metrics.BeginRun(fileSizes.Count, totalBytes); int processedCount = 0; int errorCount = 0; @@ -189,11 +195,6 @@ Func taskFunc processedPercentage, fileName ); - // Credit skipped files so byte-weighted progress can still reach 1.0 - Metrics.FileCompleted( - fileSizes.GetValueOrDefault(fileName), - TimeSpan.Zero - ); continue; } diff --git a/README.md b/README.md index 983cf224..d3ba0a20 100644 --- a/README.md +++ b/README.md @@ -841,8 +841,8 @@ PlexCleaner publishes always-on runtime metrics via `System.Diagnostics.Metrics` Read the meter with [`dotnet-counters`](https://learn.microsoft.com/dotnet/core/diagnostics/dotnet-counters): -- Local: `dotnet-counters monitor -p PlexCleaner.Process` -- Docker: `docker exec counters` (a bundled wrapper that targets the in-container process) +- Local: `dotnet-counters monitor -p --counters PlexCleaner.Process` +- Docker: `docker exec counters` (a bundled wrapper that runs the same command against the in-container process) ## Custom Plugins From b665807b4c3555992daf37f2dd8e66e6e7498659 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Sat, 18 Jul 2026 13:05:02 -0700 Subject: [PATCH 3/6] Cache State flags and use a bitwise check in EnumerateSetStates Co-Authored-By: Claude Opus 4.8 --- PlexCleaner/Metrics.cs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/PlexCleaner/Metrics.cs b/PlexCleaner/Metrics.cs index 0231cd54..2c97bd65 100644 --- a/PlexCleaner/Metrics.cs +++ b/PlexCleaner/Metrics.cs @@ -54,6 +54,14 @@ internal static class Metrics private static long s_runInflight; private static long s_runStartTimestamp; + // Cached once so the per-file RecordStates does not allocate an enum-values array (or box via + // HasFlag) on every processed file. + private static readonly SidecarFile.StatesType[] s_stateFlags = + [ + .. Enum.GetValues() + .Where(flag => flag != SidecarFile.StatesType.None), + ]; + static Metrics() { _ = s_meter.CreateObservableGauge( @@ -169,9 +177,9 @@ internal static double ComputeEtaSeconds() SidecarFile.StatesType state ) { - foreach (SidecarFile.StatesType flag in Enum.GetValues()) + foreach (SidecarFile.StatesType flag in s_stateFlags) { - if (flag != SidecarFile.StatesType.None && state.HasFlag(flag)) + if ((state & flag) == flag) { yield return flag; } From a5c638efdf51f8e8df7a3ce41c4cb57f33f07891 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Sat, 18 Jul 2026 13:14:01 -0700 Subject: [PATCH 4/6] Pre-build State tags and use bitwise flag checks to avoid per-file allocs Co-Authored-By: Claude Opus 4.8 --- PlexCleaner/Metrics.cs | 21 ++++++++++++++------- PlexCleaner/Process.cs | 2 +- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/PlexCleaner/Metrics.cs b/PlexCleaner/Metrics.cs index 2c97bd65..4015416a 100644 --- a/PlexCleaner/Metrics.cs +++ b/PlexCleaner/Metrics.cs @@ -54,12 +54,16 @@ internal static class Metrics private static long s_runInflight; private static long s_runStartTimestamp; - // Cached once so the per-file RecordStates does not allocate an enum-values array (or box via - // HasFlag) on every processed file. - private static readonly SidecarFile.StatesType[] s_stateFlags = + // Cached once: each State flag (minus None) with its pre-built tag, so the per-file RecordStates + // allocates nothing - no enum-values array, no HasFlag boxing, and no per-flag tag string. + private static readonly ( + SidecarFile.StatesType Flag, + KeyValuePair Tag + )[] s_stateTags = [ .. Enum.GetValues() - .Where(flag => flag != SidecarFile.StatesType.None), + .Where(flag => flag != SidecarFile.StatesType.None) + .Select(flag => (flag, new KeyValuePair("state", flag.ToString()))), ]; static Metrics() @@ -136,9 +140,12 @@ internal static void FileCompleted(long sizeBytes, TimeSpan wall) internal static void RecordStates(SidecarFile.StatesType state) { - foreach (SidecarFile.StatesType flag in EnumerateSetStates(state)) + foreach ((SidecarFile.StatesType flag, KeyValuePair tag) in s_stateTags) { - s_filesProcessed.Add(1, new KeyValuePair("state", flag.ToString())); + if ((state & flag) == flag) + { + s_filesProcessed.Add(1, tag); + } } } @@ -177,7 +184,7 @@ internal static double ComputeEtaSeconds() SidecarFile.StatesType state ) { - foreach (SidecarFile.StatesType flag in s_stateFlags) + foreach ((SidecarFile.StatesType flag, _) in s_stateTags) { if ((state & flag) == flag) { diff --git a/PlexCleaner/Process.cs b/PlexCleaner/Process.cs index daea4fea..873022b6 100644 --- a/PlexCleaner/Process.cs +++ b/PlexCleaner/Process.cs @@ -512,7 +512,7 @@ out string? failedOperation { Metrics.RecordModified(); } - if (state.HasFlag(SidecarFile.StatesType.VerifyFailed)) + if ((state & SidecarFile.StatesType.VerifyFailed) != 0) { Metrics.RecordVerifyFailed(); } From 3f70cb0be28a517cdef7a49c4ade8d1d70955ec6 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Sat, 18 Jul 2026 15:09:40 -0700 Subject: [PATCH 5/6] Refine runtime-metrics comments and docs - Rework the Metrics and ProcessDriver comments to house style: one statement per line, no class-header block, no prose semicolons. - Drop the internal tracking issue reference from HISTORY. - Split semicolon-joined sentences in ARCHITECTURE. - Add the 3.22 release-notes summary to README. Co-Authored-By: Claude Opus 4.8 --- ARCHITECTURE.md | 6 +++--- HISTORY.md | 4 ++-- PlexCleaner/Metrics.cs | 27 +++++++++------------------ PlexCleaner/ProcessDriver.cs | 12 +++++------- README.md | 7 +++---- 5 files changed, 22 insertions(+), 34 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 32fb7409..8068b5c7 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -77,11 +77,11 @@ All external process execution uses [CliWrap](https://github.com/Tyrrrz/CliWrap) ### Runtime Metrics -`Metrics.cs` owns a single `System.Diagnostics.Metrics.Meter` (`PlexCleaner.Process`) published for the whole process and read externally with `dotnet-counters` (no config flag; instruments are inert until observed). +`Metrics.cs` owns a single `System.Diagnostics.Metrics.Meter` (`PlexCleaner.Process`) published for the whole process and read externally with `dotnet-counters` (no config flag, and instruments are inert until observed). -- Hooks: `ProcessDriver.ProcessFiles` (the choke point every command and monitor cycle funnels through) drives the file/byte/in-flight instruments and the byte-weighted `progress.ratio`; `Process.ProcessFiles` records the per-`SidecarFile.StatesType` outcomes; `MediaTool` execution paths record `tool.duration`. +- Hooks: `ProcessDriver.ProcessFiles` (the choke point every command and monitor cycle funnels through) drives the file/byte/in-flight instruments and the byte-weighted `progress.ratio`. `Process.ProcessFiles` records the per-`SidecarFile.StatesType` outcomes. `MediaTool` execution paths record `tool.duration`. - Progress is weighted by input bytes (summed once up front and credited at completion from the same map), not file count. -- Run-scoped gauges (totals, in-flight, progress, ETA) reset per `ProcessFiles` call; the counters stay cumulative for the process. All writers use `Interlocked`, so the parallel loop needs no lock; observable-gauge callbacks only read. Tags are bounded (`state`, `tool`) - no filename tags. +- Run-scoped gauges (totals, in-flight, progress, ETA) reset per `ProcessFiles` call, while the counters stay cumulative for the process. All writers use `Interlocked`, so the parallel loop needs no lock, and observable-gauge callbacks only read. Tags are bounded (`state`, `tool`) - no filename tags. ### Sidecar File System diff --git a/HISTORY.md b/HISTORY.md index e2d24b5e..324c4316 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -5,8 +5,8 @@ Utility to optimize media files for Direct Play in Plex, Emby, Jellyfin, etc. ## Release History - Version 3.22: - - Added always-on runtime metrics published via `System.Diagnostics.Metrics` under the `PlexCleaner.Process` meter, readable with `dotnet-counters` with no extra infrastructure ([#848](https://github.com/ptr727/PlexCleaner/issues/848)). - - Overall progress is weighted by input bytes rather than file count, so a run mixing tiny and huge files reports the actual work completed; in this first version an in-flight file contributes no partial credit until it finishes. + - Added always-on runtime metrics published via `System.Diagnostics.Metrics` under the `PlexCleaner.Process` meter, readable with `dotnet-counters` with no extra infrastructure. + - Overall progress is weighted by input bytes rather than file count, so a run mixing tiny and huge files reports the actual work completed. In this first version an in-flight file contributes no partial credit until it finishes. - Instruments include the run file and byte totals, in-flight and active-thread counts, completed bytes, the weighted `progress.ratio` and `eta.seconds`, cumulative per-outcome counters (completed, modified, errors, verify-failed, and a per-`State`-flag tally), and the `file.duration` and per-tool `tool.duration` histograms. Metrics are aggregate only, with bounded `state` and `tool` tags and no filename tags. - The run-scoped gauges reset at the start of every processing pass, so monitor mode and back-to-back commands each report their own run, while the counters stay cumulative for rate display. - Instruments are inert until a listener observes them, so the feature is always on with no configuration flag and negligible idle overhead. diff --git a/PlexCleaner/Metrics.cs b/PlexCleaner/Metrics.cs index 4015416a..364ecf8f 100644 --- a/PlexCleaner/Metrics.cs +++ b/PlexCleaner/Metrics.cs @@ -3,17 +3,11 @@ namespace PlexCleaner; -// Always-on runtime metrics published via System.Diagnostics.Metrics, readable with dotnet-counters -// (no config flag; instruments are inert until a listener observes them). Overall progress is -// weighted by input bytes, not file count, so a run with one small and one huge file reports real -// work done. Aggregate only: no filename tags (unbounded cardinality) - the state and tool tags are -// bounded enums. internal static class Metrics { private static readonly Meter s_meter = new("PlexCleaner.Process"); - // Cumulative counters live for the whole process (correct for dotnet-counters rate display) and - // are NOT reset between runs or monitor cycles. + // Cumulative for the process lifetime, not reset between runs. private static readonly Counter s_filesCompleted = s_meter.CreateCounter( "plexcleaner.files.completed", description: "Files finished, any outcome" @@ -46,16 +40,15 @@ internal static class Metrics description: "Per media-tool invocation time, tagged by tool" ); - // Run-scoped state, reset by BeginRun and only read by the observable gauges. All access is via - // Interlocked so the parallel per-file loop needs no lock. + // Run-scoped state, reset by BeginRun, read by the observable gauges. + // All access is via Interlocked so the parallel loop needs no lock. private static long s_runFilesTotal; private static long s_runBytesTotal; private static long s_runBytesCompleted; private static long s_runInflight; private static long s_runStartTimestamp; - // Cached once: each State flag (minus None) with its pre-built tag, so the per-file RecordStates - // allocates nothing - no enum-values array, no HasFlag boxing, and no per-flag tag string. + // Each State flag (minus None) with its tag, pre-built once so RecordStates allocates nothing per file. private static readonly ( SidecarFile.StatesType Flag, KeyValuePair Tag @@ -108,8 +101,7 @@ static Metrics() ); } - // Start a new run: reset the run-scoped gauges and the ETA clock. Called at every - // ProcessDriver.ProcessFiles entry, so back-to-back commands and each monitor cycle restart clean. + // Reset the run-scoped gauges and ETA clock, called once per ProcessFiles run. internal static void BeginRun(long totalFiles, long totalBytes) { _ = Interlocked.Exchange(ref s_runFilesTotal, totalFiles); @@ -123,8 +115,7 @@ internal static void BeginRun(long totalFiles, long totalBytes) internal static void FileInflightDone() => Interlocked.Decrement(ref s_runInflight); - // A file finished (any outcome): credit its whole size (no partial credit in v1) and record its - // wall-clock time. + // A finished file credits its whole size (no partial credit in v1) and its wall-clock time. internal static void FileCompleted(long sizeBytes, TimeSpan wall) { _ = Interlocked.Add(ref s_runBytesCompleted, sizeBytes); @@ -157,15 +148,15 @@ internal static void RecordToolDuration(MediaTool.ToolType tool, double millisec internal static void Dispose() => s_meter.Dispose(); - // Byte-weighted progress; guards a zero (or not-yet-started) total. + // Byte-weighted progress, guards a zero (or not-yet-started) total. internal static double ComputeProgress() { long total = Interlocked.Read(ref s_runBytesTotal); return total <= 0 ? 0.0 : (double)Interlocked.Read(ref s_runBytesCompleted) / total; } - // Linear extrapolation from weighted progress and elapsed time; 0 until there is progress, and - // never a non-finite value (which dotnet-counters would render as NaN/Infinity). + // Linear extrapolation from weighted progress and elapsed time. + // Returns 0 before any progress and never a non-finite value. internal static double ComputeEtaSeconds() { double ratio = ComputeProgress(); diff --git a/PlexCleaner/ProcessDriver.cs b/PlexCleaner/ProcessDriver.cs index ef74a1a9..25440725 100644 --- a/PlexCleaner/ProcessDriver.cs +++ b/PlexCleaner/ProcessDriver.cs @@ -118,15 +118,13 @@ Func taskFunc // Process all files in parallel int totalCount = fileList.Count; - // Size map for the byte-weighted progress metrics: sum once up front and credit the same size - // at completion, so a remux or rename mid-run cannot drift the total. Missing files weight as - // zero. BeginRun resets the run-scoped gauges, covering back-to-back commands and monitor cycles. + // Sum sizes up front and credit the same size at completion, so a mid-run rename cannot drift the total. + // Missing files weight as zero. Dictionary fileSizes = new(totalCount, StringComparer.Ordinal); long totalBytes = 0; foreach (string file in fileList) { - // Weight only the files that will actually be processed: a mkvFilesOnly command skips - // non-MKV files, so they are excluded from the totals and never credited as completed. + // Exclude non-MKV files from the totals when mkvFilesOnly, since they are skipped not processed. if (mkvFilesOnly && !SidecarFile.IsMkvFile(file)) { continue; @@ -205,8 +203,8 @@ Func taskFunc fileName ); - // Perform the task, timing this file's work. Track the in-flight count around - // the task in a finally so a cancellation cannot leak it. + // Perform the task, timing this file's work. + // Decrement the in-flight count in a finally so a cancellation cannot leak it. long fileSize = fileSizes.GetValueOrDefault(fileName); Metrics.FileStarted(); long startTimestamp = Stopwatch.GetTimestamp(); diff --git a/README.md b/README.md index d3ba0a20..bcc54b2a 100644 --- a/README.md +++ b/README.md @@ -23,13 +23,12 @@ Utility to optimize media files for Direct Play in Plex, Emby, Jellyfin, etc. ### Release Notes -**Version: 3.21**: +**Version: 3.22**: **Summary:** -- Treat a non-monotonic DTS as a verify failure, and attempt to repair it losslessly with the `setts` bitstream filter. -- Switched closed caption detection to `ffprobe -analyze_frames`, and consolidated the bitrate and DTS packet analyses into a single packet pass. -- Added the `DtsTimestampRepair` example plugin that attempts non-monotonic DTS repairs on `RepairFailed` files. +- Added always-on runtime metrics with byte-weighted progress, published via `System.Diagnostics.Metrics` and readable with `dotnet-counters`. +- Bundled a `counters` wrapper in the Docker image for reading the metrics with a single command. See [Release History](./HISTORY.md) for complete release notes and older versions. From 3909e208da8e74c3a1aa948f9589c8cc95a681b4 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Sat, 18 Jul 2026 16:43:40 -0700 Subject: [PATCH 6/6] Cache tool tags to avoid a per-invocation alloc in RecordToolDuration Co-Authored-By: Claude Opus 4.8 --- PlexCleaner/Metrics.cs | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/PlexCleaner/Metrics.cs b/PlexCleaner/Metrics.cs index 364ecf8f..63baf166 100644 --- a/PlexCleaner/Metrics.cs +++ b/PlexCleaner/Metrics.cs @@ -59,6 +59,16 @@ private static readonly ( .Select(flag => (flag, new KeyValuePair("state", flag.ToString()))), ]; + // Each tool with its pre-built tag, so RecordToolDuration allocates nothing per invocation. + private static readonly Dictionary< + MediaTool.ToolType, + KeyValuePair + > s_toolTags = Enum.GetValues() + .ToDictionary( + tool => tool, + tool => new KeyValuePair("tool", tool.ToString()) + ); + static Metrics() { _ = s_meter.CreateObservableGauge( @@ -141,10 +151,7 @@ internal static void RecordStates(SidecarFile.StatesType state) } internal static void RecordToolDuration(MediaTool.ToolType tool, double milliseconds) => - s_toolDuration.Record( - milliseconds, - new KeyValuePair("tool", tool.ToString()) - ); + s_toolDuration.Record(milliseconds, s_toolTags[tool]); internal static void Dispose() => s_meter.Dispose();