From 261a1948cb1b03521cd3141c4b2f88beb122e14a Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Fri, 17 Jul 2026 08:56:17 -0700 Subject: [PATCH 01/12] Add Regression Test Harness and Corpus Tooling Scripts Add the ZFS-clone regression harness and the standalone Python tooling that verifies PlexCleaner's processing decisions stay consistent across versions against a curated media collection: catalog generation, reduced-corpus build with a prove-equivalence gate, decode-signature location, and physical-shape coverage audit. No media or media filenames are committed. Media-specific reduction rules are externalized to a JSON file that lives with the media and is read and written by the tooling; the repo ships only a synthetic example. Co-Authored-By: Claude Opus 4.8 --- RegressionTests/RegressionTest.sh | 384 +++++++++++++++++++ RegressionTests/audit_physical.py | 114 ++++++ RegressionTests/catalog_corpus.py | 99 +++++ RegressionTests/corpus_common.py | 364 ++++++++++++++++++ RegressionTests/locate_issue.py | 221 +++++++++++ RegressionTests/reduce_corpus.py | 380 ++++++++++++++++++ RegressionTests/reduction-rules.example.json | 16 + 7 files changed, 1578 insertions(+) create mode 100755 RegressionTests/RegressionTest.sh create mode 100644 RegressionTests/audit_physical.py create mode 100644 RegressionTests/catalog_corpus.py create mode 100644 RegressionTests/corpus_common.py create mode 100644 RegressionTests/locate_issue.py create mode 100644 RegressionTests/reduce_corpus.py create mode 100644 RegressionTests/reduction-rules.example.json diff --git a/RegressionTests/RegressionTest.sh b/RegressionTests/RegressionTest.sh new file mode 100755 index 00000000..25f8b6e2 --- /dev/null +++ b/RegressionTests/RegressionTest.sh @@ -0,0 +1,384 @@ +#!/bin/bash + +# Regression test harness: process the ZFS test dataset through a single PlexCleaner Docker +# image tag and write per-version results/logs for diffing against other runs. +# +# Usage: RegressionTest.sh [quick|full] [tag] [corpus] [plugin...] +# quick (default) full-file scan as in production, remux/re-encode to short testsnippets +# full also remux/re-encode the complete media (slowest) +# tag docker image tag to test, default develop; the runner specifies one tag per run +# corpus full (default) or reduced - which corpus subdirectory of the test clone to process +# plugin optional plugin name(s) from the registry to run after processing, default MatroskaHeaderCleanup +# +# Test data provisioning (2026-07-16): the corpus lives in the snapshotted dataset +# hddpool/media/troublesome (full/ + reduced/); each run provisions hddpool/media/test as an +# instant zero-copy ZFS CLONE of the newest corpus snapshot (no rsync, no drift - the clone +# replaces the old PrepTestDataset.sh flow). Rollbacks inside the run use the clone's @backup. +# +# Results are written into a directory named after the image build version (the "version" +# label, e.g. 3.20.7-g3235a8a055 - the same tag used for source code sync and plugin build). +# This makes results durable and version-to-version comparable, and replaces the old manual +# step of copying results into a version-named folder when happy with a release. Re-running +# the same build overwrites the files in that build's directory. +# +# The flow is: +# 1. pull the image +# 2. read the build version from the image "version" label +# 3. create the version-named directory under this folder +# 4. copy the live PlexCleaner.json settings into the directory (durable record) +# 5. write buildinfo.json (channel, version, image digest, run timestamp) +# 6. build the requested plugins from the matching source tag into the directory +# 7. run process + defaultsettings, then each plugin on the processed results, saving into the directory +# +# Settings file PlexCleaner.json is expected to use: +# "UseSystem": true, "AutoUpdate": false, "RemoveUnwantedLanguageTracks": true, +# "RemoveDuplicateTracks": true, "DeInterlace": true +# Test data provisioned per run as a ZFS clone (see above); PrepTestDataset.sh is retired. +# +# The plugin assembly is not shipped in the image, so the script clones the PlexCleaner source, +# checks out the git tag matching the image version label (the release tag), and builds the example +# MatroskaHeaderCleanup plugin from it, so the plugin API matches the running image. Requires git and +# the .NET SDK on the host; the custom run is skipped with a warning if they are missing or the version +# tag cannot be checked out. + +set -euxo pipefail + +# Run as root to allow ZFS snapshot rollback +if [[ "$(id -u)" -ne 0 ]]; then + echo "This script must be run as root" >&2 + exit 1 +fi + +# Test mode, default quick +Mode="${1:-quick}" +case "$Mode" in + quick | full) ;; + *) + echo "Usage: $0 [quick|full] [tag]" >&2 + exit 1 + ;; +esac + +# Paths +PlexCleanerApp="/PlexCleaner/Debug/PlexCleaner" +MediaPath="/Test/Media" +ConfigPath="/Test/Config" +HostMedia="/data/media/test" +HostConfig="/data/media/PlexCleaner/RegressionTest" +CorpusDataset="hddpool/media/troublesome" +TestDataset="hddpool/media/test" +Snapshot="hddpool/media/test@backup" +Image="docker.io/ptr727/plexcleaner" +# Live master settings copied into each build's version directory +Settings="PlexCleaner.json" +# Example plugins runnable after the process test, name -> "project dll". The post-process plugin runs +# only confirm a plugin loads and runs against the processed dataset; per-plugin behaviour is verified +# in isolation elsewhere, not here. +declare -A PluginRegistry=( + [MatroskaHeaderCleanup]="Plugins/MatroskaHeaderCleanup/MatroskaHeaderCleanup.csproj MatroskaHeaderCleanup.dll" + [DtsTimestampRepair]="Plugins/DtsTimestampRepair/DtsTimestampRepair.csproj DtsTimestampRepair.dll" +) +DefaultPlugin="MatroskaHeaderCleanup" +PluginRepo="https://github.com/ptr727/PlexCleaner.git" +PluginBuildDir="/data/media/PlexCleaner/PluginBuild" + +# Parallel file-processing thread count; server has ample cores, so double the default of 4 +ThreadCount=8 + +# Process options, always run in parallel +# quick mode keeps full-file scanning but writes short testsnippets to shorten remux and re-encode +ProcessOptions=(--parallel --threadcount "$ThreadCount") +if [[ "$Mode" == "quick" ]]; then + ProcessOptions+=(--testsnippets) +fi + +# Common docker run arguments +# Allocate a TTY only when attached to one, so the script also runs non-interactively (CI, background) +TtyArg=() +[[ -t 0 ]] && TtyArg=(-it) +DockerCommon=( + "${TtyArg[@]}" + --rm + --name PlexCleaner-RegressionTest + --user nobody:users + --env TZ=America/Los_Angeles +) + +# Provision the test dataset as a zero-copy clone of the newest corpus snapshot. +# Fast path (every run): the existing clone already originates from the newest snapshot -> just +# roll back to its @backup (rollback works even while long-running media containers or SMB +# clients hold the mount in their namespaces; destroy does NOT - Plex/Jellyfin/smbd bind +# /data/media at start). +# Corpus-change path: rename the stale clone aside (rename succeeds where destroy is blocked), +# clone fresh, and opportunistically destroy any retired clones once the holders are gone. +ProvisionDataset() { + local CorpusSnap Origin + CorpusSnap="$(zfs list -H -t snapshot -o name -s creation "$CorpusDataset" | tail -1)" + if [[ -z "$CorpusSnap" ]]; then + echo "No snapshot found on $CorpusDataset - snapshot the corpus first" >&2 + exit 1 + fi + + Origin="$(zfs get -H -o value origin "$TestDataset" 2>/dev/null || true)" + if [[ "$Origin" == "$CorpusSnap" ]]; then + echo "Test clone current ($CorpusSnap) - rolling back" + sudo zfs rollback "$Snapshot" + return + fi + + echo "Provisioning $TestDataset as a clone of $CorpusSnap" + if zfs list "$TestDataset" >/dev/null 2>&1; then + sudo zfs rename "$TestDataset" "$TestDataset-retired-$(date +%s)" + fi + sudo zfs clone "$CorpusSnap" "$TestDataset" + sudo zfs snapshot "$Snapshot" + + # best-effort cleanup of retired clones (succeeds once the holders have restarted/closed) + local Retired + for Retired in $(zfs list -H -o name 2>/dev/null | grep -E "^$TestDataset-retired-" || true); do + sudo zfs destroy -r "$Retired" 2>/dev/null && + echo "Destroyed retired clone $Retired" || + echo "Retired clone $Retired still held; will retry next run" >&2 + done +} + +# Restore the test dataset (clone) to its pristine post-provision state +RestoreDataset() { + echo "Restoring test dataset" + sudo zfs rollback "$Snapshot" +} + +# GetImageVersion Tag -> prints the build version from the image "version" label +GetImageVersion() { + local Tag="$1" + docker image inspect "$Image:$Tag" --format '{{ index .Config.Labels "version" }}' | tr -d '\r' +} + +# WriteBuildInfo Tag Version VersionDir +# Record the channel and run metadata so results can be identified and ordered later, +# e.g. "the last develop build" vs "the last latest build". +WriteBuildInfo() { + local Tag="$1" + local Version="$2" + local VersionDir="$3" + + local ImageId Digest Created Now + ImageId="$(docker image inspect "$Image:$Tag" --format '{{ .Id }}' | tr -d '\r')" + Digest="$(docker image inspect "$Image:$Tag" --format '{{ if .RepoDigests }}{{ index .RepoDigests 0 }}{{ end }}' | tr -d '\r')" + Created="$(docker image inspect "$Image:$Tag" --format '{{ .Created }}' | tr -d '\r')" + Now="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + + cat >"$VersionDir/buildinfo.json" </dev/null || ! command -v dotnet >/dev/null; then + echo "Skipping plugin build: git and the .NET SDK are required on the host" >&2 + return 1 + fi + + echo "Building $Dll from $Image:$Tag at tag $Version" + + # Clone once, then fetch tags and check out the release tag matching the image version + if [[ ! -d "$PluginBuildDir/.git" ]]; then + git clone "$PluginRepo" "$PluginBuildDir" || return 1 + fi + git -C "$PluginBuildDir" fetch --force --tags origin || return 1 + git -C "$PluginBuildDir" checkout --force "$Version" || return 1 + + # Build the plugin (Debug matches the image binary under /PlexCleaner/Debug) + dotnet build "$PluginBuildDir/$Project" --configuration Debug || return 1 + + # Copy the built assembly into the version directory for the custom run + local Built + Built="$(find "$PluginBuildDir/.artifacts/bin" -name "$Dll" -print -quit)" + if [[ -z "$Built" ]]; then + echo "Skipping plugin build: built assembly $Dll not found" >&2 + return 1 + fi + cp "$Built" "$VersionDir/$Dll" +} + +# RunPlugin Tag Version Dll +# Run the custom command with one plugin against the current (already processed) dataset, no rollback, +# to confirm the plugin loads and runs. Per-plugin log so multiple plugin runs do not overwrite each other +RunPlugin() { + local Tag="$1" + local Version="$2" + local Dll="$3" + local ConfigDir="$ConfigPath/$Version" + [[ "$Corpus" == "reduced" ]] && ConfigDir="$ConfigPath/$Version-reduced" + + echo "Running PlexCleaner custom plugin $Dll on $Image:$Tag" + docker run \ + "${DockerCommon[@]}" \ + --volume "$HostMedia:$MediaPath:rw" \ + --volume "$HostConfig:$ConfigPath:rw" \ + "$Image:$Tag" \ + "$PlexCleanerApp" custom \ + --settingsfile="$ConfigDir/$Settings" \ + --logfile="$ConfigDir/PlexCleaner_custom_${Dll%.dll}.log" \ + "${LogOptions[@]}" \ + --mediafiles="$MediaPath/$Corpus" \ + --pluginassembly="$ConfigDir/$Dll" \ + --parallel --threadcount "$ThreadCount" +} + +# RunDefaultSettings Tag Version +RunDefaultSettings() { + local Tag="$1" + local Version="$2" + local ConfigDir="$ConfigPath/$Version" + [[ "$Corpus" == "reduced" ]] && ConfigDir="$ConfigPath/$Version-reduced" + + echo "Running PlexCleaner defaultsettings on $Image:$Tag" + docker run \ + "${DockerCommon[@]}" \ + --volume "$HostConfig:$ConfigPath:rw" \ + "$Image:$Tag" \ + "$PlexCleanerApp" defaultsettings \ + --settingsfile="$ConfigDir/PlexCleaner.defaults.json" +} + +# RunCreateSchema Tag Version +RunCreateSchema() { + local Tag="$1" + local Version="$2" + local ConfigDir="$ConfigPath/$Version" + [[ "$Corpus" == "reduced" ]] && ConfigDir="$ConfigPath/$Version-reduced" + + echo "Running PlexCleaner createschema on $Image:$Tag" + docker run \ + "${DockerCommon[@]}" \ + --volume "$HostConfig:$ConfigPath:rw" \ + "$Image:$Tag" \ + "$PlexCleanerApp" createschema \ + --schemafile="$ConfigDir/PlexCleaner.schema.json" +} + +# RunRegressionTests Tag +RunRegressionTests() { + local Tag="$1" + + docker pull "$Image:$Tag" + + # Name the results directory after the image build version (used for source sync too) + local Version + Version="$(GetImageVersion "$Tag")" + if [[ -z "$Version" ]]; then + echo "Could not read the version label from $Image:$Tag" >&2 + exit 1 + fi + echo "Build version for $Image:$Tag is $Version" + + # Create the version directory, overwriting existing files on a re-run of the same build + # reduced-corpus runs get their own results directory so they never clobber the full baseline + local VersionDir="$HostConfig/$Version" + [[ "$Corpus" == "reduced" ]] && VersionDir="$HostConfig/$Version-reduced" + mkdir -p "$VersionDir" + + # Copy the live settings into the version directory for a durable record of what was run + cp "$HostConfig/$Settings" "$VersionDir/$Settings" + + WriteBuildInfo "$Tag" "$Version" "$VersionDir" + + # v3.20+ appends to the log file by default and supports --logclear to clear on re-run + # Debug level logs each tool invocation with its command line, so a failure can be reproduced directly + local LogOptions=(--logclear --loglevel Debug) + + # Build each requested plugin from the source tag matching the image; skip any that fail to build + local Plugin Entry Project Dll + local -a BuiltPlugins=() + for Plugin in "${Plugins[@]}"; do + Entry="${PluginRegistry[$Plugin]:-}" + if [[ -z "$Entry" ]]; then + echo "Skipping unknown plugin: $Plugin" >&2 + continue + fi + read -r Project Dll <<<"$Entry" + if BuildPlugin "$Tag" "$Version" "$VersionDir" "$Project" "$Dll"; then + BuiltPlugins+=("$Dll") + else + echo "Skipping plugin run, build failed: $Plugin" >&2 + fi + done + + RunDefaultSettings "$Tag" "$Version" + # RunCreateSchema "$Tag" "$Version" + + # Provision a pristine clone of the corpus, process it, then run each built plugin on the + # processed results to confirm it loads and runs; per-plugin behaviour is verified in isolation + ProvisionDataset + RunProcess "$Tag" "$Version" + for Dll in "${BuiltPlugins[@]}"; do + RunPlugin "$Tag" "$Version" "$Dll" + done + RestoreDataset +} + +echo "Starting tests in $Mode mode" + +# Single image tag to test (default develop), corpus selection (full|reduced subdirectory of the +# test clone, default full), and optional plugins to run after processing (default the +# MatroskaHeaderCleanup example), e.g. +# RegressionTest.sh quick develop reduced DtsTimestampRepair MatroskaHeaderCleanup +Tag="${2:-develop}" +Corpus="${3:-full}" +case "$Corpus" in + full | reduced) ;; + *) + echo "Usage: $0 [quick|full] [tag] [full|reduced] [plugin...]" >&2 + exit 1 + ;; +esac +Plugins=("${@:4}") +[[ ${#Plugins[@]} -eq 0 ]] && Plugins=("$DefaultPlugin") +RunRegressionTests "$Tag" + +echo "Done with tests" diff --git a/RegressionTests/audit_physical.py b/RegressionTests/audit_physical.py new file mode 100644 index 00000000..42b5cfee --- /dev/null +++ b/RegressionTests/audit_physical.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +""" +audit_physical.py - physical-error-shape audit of the reduced corpus. + +The reduce gate compares States, detections and broad signature CLASSES; a class can lump several +distinct physical ffmpeg messages, so a clip could carry a different physical defect that maps to +the same class - and a future, more precise PlexCleaner classification would then invalidate the +sample. This audit closes that gap: it extracts every physical error SHAPE (exact message +template, run- and site-varying content normalized) from the ground run and from each reduced +clip's processing log, and reports any source shape the clip does not reproduce. + +Augments the reduced collection's catalog.json in place with per-file: + source_error_shapes / clip_error_shapes / missing_error_shapes / shape_coverage +Kept-full and verbatim entries are equal by construction (the shipped file IS the source). +""" + +import argparse +import json +from collections.abc import Iterable +from pathlib import Path + +from corpus_common import error_shape, find_run, parse_log, stem_of + +# Default paths for the reference server; override on the command line for another environment. +SCRATCH = Path("/data/media/PlexCleaner/scratch-trim") + + +def shapes_of(errors: Iterable[str]) -> list[str]: + return sorted({error_shape(e) for e in errors if e != ""}) + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--run", help="ground-truth run dir (default: newest develop run)") + ap.add_argument( + "--reduced", + default="/data/media/troublesome/reduced", + help="reduced collection dir holding catalog.json", + ) + ap.add_argument( + "--work", + default=str(SCRATCH / "work"), + help="root of the per-clip processing logs from the reduce run", + ) + args = ap.parse_args() + + reduced = Path(args.reduced) + work = Path(args.work) + run = find_run(args.run) + ground = parse_log(run / "PlexCleaner_process.log") + collection = json.loads((reduced / "catalog.json").read_text()) + manifest = collection["files"] + + gaps = 0 + audited = 0 + for e in manifest: + stem = stem_of(e["file"]) + src_shapes = shapes_of(ground.get(stem, {"errors": set()})["errors"]) + e["source_error_shapes"] = src_shapes + + if e["decision"] != "reduced" or e.get("method") == "verbatim": + # the shipped file IS the source; physically identical by construction + e["clip_error_shapes"] = src_shapes + e["missing_error_shapes"] = [] + continue + + clip_log = work / stem / "out" / "clip_process.log" + if not clip_log.exists(): + e["clip_error_shapes"] = None + e["missing_error_shapes"] = [""] + gaps += 1 + continue + audited += 1 + cl_shapes = shapes_of(parse_log(clip_log).get(stem, {"errors": set()})["errors"]) + missing = sorted(set(src_shapes) - set(cl_shapes)) + e["clip_error_shapes"] = cl_shapes + e["missing_error_shapes"] = missing + if missing: + gaps += 1 + print(f"GAP {e['file'][:66]}") + for s in missing: + print(f" - {s[:130]}") + + # Quantified completeness: the corpus does not need to be perfect, it needs to be MEASURED. + # Per-file coverage + a corpus-level figure make the residual gap explicit, so it is always + # known when a change touches an under-covered area and a full-corpus run is warranted. + total_src = total_hit = 0 + for e in manifest: + src = set(e["source_error_shapes"]) + clip = set(e["clip_error_shapes"] or []) + e["shape_coverage"] = round(len(src & clip) / len(src), 3) if src else 1.0 + total_src += len(src) + total_hit += len(src & clip) + + (reduced / "catalog.json").write_text(json.dumps(collection, indent=2, ensure_ascii=False)) + with_src = sum(1 for e in manifest if e["source_error_shapes"]) + print(f"\naudited {audited} cut clips ({with_src} files have source error shapes at all)") + print(f"files with physical-shape gaps: {gaps}") + print( + f"CORPUS PHYSICAL-SHAPE COVERAGE: {total_hit}/{total_src} shapes = " + f"{100 * total_hit / total_src:.1f}%" + if total_src + else "no source shapes" + ) + inc = [(e["file"], e["shape_coverage"]) for e in manifest if e["shape_coverage"] < 1.0] + if inc: + print("incomplete files (full-corpus run needed for changes touching these):") + for f, c in sorted(inc, key=lambda x: x[1]): + print(f" {c * 100:5.1f}% {f[:66]}") + print(f"catalog augmented: {reduced / 'catalog.json'}") + + +if __name__ == "__main__": + main() diff --git a/RegressionTests/catalog_corpus.py b/RegressionTests/catalog_corpus.py new file mode 100644 index 00000000..6fee131c --- /dev/null +++ b/RegressionTests/catalog_corpus.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +""" +catalog_corpus.py - derive a reproducible, machine-readable issue catalog for a media corpus +from a versioned PlexCleaner regression run. + +Generated, never hand-maintained: re-run it and it re-derives every file's issue set from the +actual tool output of a chosen run. Output is `catalog.json` ONLY (automation; no human README); +it is the source of truth for what each file must reproduce and the target the reduced (quickscan) +corpus is validated against. Shared parsing/classification lives in corpus_common.py. + +Usage: catalog_corpus.py [--run ] [--out catalog.json] +""" + +import argparse +import json +import os +from collections import Counter +from pathlib import Path + +from corpus_common import ( + MEDIA_EXTS, + SRC_DIR, + buckets_for, + classify_signature, + find_run, + parse_log, + stem_of, +) + + +def build(run): + res = json.loads((run / "Results_process.json").read_text()) + versions = res.get("Versions", {}) + log = parse_log(run / "PlexCleaner_process.log") + err_files = {stem_of(x) for x in res["Results"]["Errors"]["Files"]} + vf_files = {stem_of(x) for x in res["Results"]["VerifyFailed"]["Files"]} + + entries = [] + for r in res["Results"]["Results"]: + name = os.path.basename(r["OriginalFileName"]) + if Path(name).suffix.lower() not in MEDIA_EXTS: + continue + stem = stem_of(name) + state = set(s.strip() for s in (r.get("State") or "").split(",") if s.strip()) + lg = log.get(stem, {"detections": set(), "errors": set(), "tracks": []}) + sig = classify_signature(lg["errors"]) + buckets = buckets_for(state, lg["detections"], sig, stem in err_files) + # FileDeleted / consumed samples yield no derivable output; fall back to a marker bucket + if not buckets and not state: + buckets = {"FileDeleted"} + entries.append( + { + "file": name, + "buckets": sorted(buckets), + "state": sorted(state), + "result": r.get("Result"), + "modified": r.get("Modified"), + "in_errors": stem in err_files, + "in_verifyfailed": stem in vf_files, + "detections": sorted(lg["detections"]), + "verify_errors": sorted(lg["errors"]), + "decode_subtypes": sorted(sig), + "tracks": lg["tracks"], + } + ) + entries.sort(key=lambda e: e["file"]) + return { + "schema": 1, + "source_run": run.name, + "application": versions.get("Application"), + "tools": { + t.get("ToolType", t.get("ToolFamily", "?")): t.get("Version") + for t in versions.get("Tools", []) + }, + "file_count": len(entries), + "files": entries, + } + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--run", help="version dir (default newest develop run)") + ap.add_argument("--out", default=str(SRC_DIR / "catalog.json")) + args = ap.parse_args() + + run = find_run(args.run) + catalog = build(run) + Path(args.out).write_text(json.dumps(catalog, indent=2, ensure_ascii=False)) + + print(f"run={run.name} app={catalog['application']} files={catalog['file_count']}") + print(f"catalog -> {args.out}") + bc = Counter(b for e in catalog["files"] for b in e["buckets"]) + print("bucket distribution:") + for b, n in bc.most_common(): + print(f" {n:3} {b}") + + +if __name__ == "__main__": + main() diff --git a/RegressionTests/corpus_common.py b/RegressionTests/corpus_common.py new file mode 100644 index 00000000..c48121c1 --- /dev/null +++ b/RegressionTests/corpus_common.py @@ -0,0 +1,364 @@ +""" +corpus_common.py - shared parsing + deterministic classification for the corpus tooling. + +Single source of truth so catalog_corpus.py and reduce_corpus.py never drift. Parses a +PlexCleaner process log (attributing detections/errors to files by thread affinity + line +filename) and classifies a file's State/detections/decode-signatures into a curated stable +bucket set. + +Robust to BOTH failure-log formats: a logging change inserted {Operation} before ExitCode and +appended {FileName} after the error, so a fixed regex tied to the old shape would silently drop +all errors on newer logs. +""" + +import json +import re +import sys +from pathlib import Path + +REG_DIR = Path("/data/media/PlexCleaner/RegressionTest") +SRC_DIR = Path("/data/media/troublesome/full") + +MEDIA_EXTS = { + ".mkv", + ".mp4", + ".avi", + ".wmv", + ".mov", + ".ts", + ".mpg", + ".m2ts", + ".dv", + ".webm", + ".m4v", +} + +# Robust to both log timestamp formats: +# short : "10:41:46 [INF] <9> msg" +# debug : "2026-07-15 10:41:46.977 -07:00 [INF] <9> msg" (--loglevel Debug full timestamp) +# The old short-only pattern silently matched nothing on debug logs, dropping every detection/error +# and making the equivalence check vacuous (empty >= empty). +LINE_RE = re.compile( + r"^(?:\d{4}-\d\d-\d\d )?\d\d:\d\d:\d\d(?:\.\d+)?(?: [+-]\d\d:\d\d)? \[\w+\] <(\d+)> (.*)$" +) +BEFORE_RE = re.compile(r'ProcessFiles.*?(?:Before|Skipping non-MKV file)\s*:\s*"([^"]+)"') +DETECT_RE = re.compile(r"([A-Z][A-Za-z0-9 /_-]+?) detected\b") +TRACK_RE = re.compile( + r'MkvMerge\s*:\s*(Video|Audio|Subtitle)\s*:\s*Format:\s*"([^"]*)".*?Interlaced:\s*(True|False)' +) +# old : Failed execution of FfMpeg : ExitCode: 183 : "e1 | e2 | e3" +# new : Failed execution of FfMpeg : Verify : ExitCode: 183 : "e1 | e2 | e3" : "/path/file.mkv" +FAIL_RE = re.compile(r"Failed execution of (\w+)\b.*?ExitCode:\s*-?\d+\s*(?::\s*(.*))?$") +QUOTED_RE = re.compile(r'"([^"]*)"') +# Environmental / informational stderr that is NOT a file-intrinsic issue - a reduced clip must not be +# required to reproduce these (they depend on the host, not the media). +BENIGN_NOISE = re.compile( + r"Cannot load lib|libnvidia|libcuda|Using the demultiplexer|Using the muxer for|" + r"Using the encoder|Using the decoder|Press \[q\]|configuration:|built with|" + r"deprecated pixel format", + re.IGNORECASE, +) + +DECODE_SIGNATURES = [ + ("non monotonically increasing dts", "DTS-NonMonotonic"), + ("Invalid NAL unit size", "Decode-NAL"), + ("mmco", "Decode-H264-RefPicture"), + ("reference picture missing", "Decode-H264-RefPicture"), + ("Missing reference picture", "Decode-H264-RefPicture"), + ("number of reference frames", "Decode-H264-RefFrames"), + ("cabac decode", "Decode-H264-Cabac"), + ("error while decoding", "Decode-Generic"), + # NOTE: "Invalid data found when processing input" is ffmpeg's blanket wrapper accompanying any + # hard decode failure, not a distinct issue class - classifying it forced clips to reproduce + # the wrapper severity rather than the actual signature, so it is intentionally absent. + ("noise_facs_q", "Decode-AAC"), + ("env_facs_q", "Decode-AAC"), + ("Input buffer exhausted", "Decode-AAC"), + ("Unknown subtitle segment", "Subtitle-Corrupt"), + ("quant_step_size", "TrueHD-QuantStep"), + ("Output file is empty", "Encode-EmptyOutput"), + ("non-supported file type", "Container-Unsupported"), + ("exceeds max length", "EBML-MaxLength"), +] + +STATE_BUCKET = { + "DeInterlaced": "Interlaced", + "ClearedCaptions": "ClosedCaption", + "BitrateExceeded": "Bitrate", + "SetLanguage": "Language", + "SetFlags": "Flags", + "ClearedDefaultFlags": "Flags", + "ClearedTags": "Tags", + "RemovedAttachments": "Attachments", + "RemovedCoverArt": "CoverArt", + "ReEncoded": "ReEncode", + "Repaired": "VerifyRepair", + "VerifyFailed": "VerifyFailed", + "FileReNamed": "ExtensionNormalize", +} +DETECT_BUCKET = [ + ("Interlaced", "Interlaced"), + ("Closed Caption", "ClosedCaption"), + ("Cover Art", "CoverArt"), + ("Attachment", "Attachments"), + ("language", "Language"), + ("Default flags", "Flags"), + ("flags to be set", "Flags"), + ("Tags", "Tags"), + ("Metadata", "Tags"), + ("encode", "ReEncode"), + ("Verify", "VerifyRepair"), + ("Duplicate", "DuplicateTracks"), + ("Extra video", "ExtraTracks"), +] + + +def stem_of(p): + return Path(p).stem + + +def hms(seconds): + """Seconds -> HH:MM:SS (mkvmerge rejects a seconds field > 59, so never emit 00:00:60).""" + seconds = int(seconds) + return f"{seconds // 3600:02d}:{(seconds % 3600) // 60:02d}:{seconds % 60:02d}" + + +def strip_language_ietf(path): + """Surgical defect RE-INJECTION: delete the LanguageIETF element from every track header, in + place (no remux). A mkvmerge cut writes IETF tags, repairing the 'Metadata errors' defect many + sources carry; deleting them restores the defect so the clip exercises the same repair path. + Track languages themselves are untouched.""" + import subprocess as _sp + + path = Path(path) + try: + ident = json.loads( + _sp.run( + ["mkvmerge", "-J", str(path)], stdin=_sp.DEVNULL, capture_output=True, text=True + ).stdout + ) + ntracks = len(ident.get("tracks", [])) + except Exception: + return False + if not ntracks: + return False + cmd = ["mkvpropedit", str(path)] + for i in range(1, ntracks + 1): + cmd += ["--edit", f"track:@{i}", "--delete", "language-ietf"] + r = _sp.run(cmd, stdin=_sp.DEVNULL, stdout=_sp.DEVNULL, stderr=_sp.DEVNULL) + return r.returncode == 0 + + +def set_language_ietf(path): + """Inverse surgery of strip_language_ietf: SET language-ietf on every track (from the track's + existing language), in place. An ffmpeg cut strips IETF tags, which triggers PlexCleaner's + metadata remux BEFORE verify - and that remux repairs timestamp defects the clip was built to + carry. Fixing IETF up front lets the clip enter verify metadata-clean, so the defect drives + the same verify->repair chain as the source did.""" + import subprocess as _sp + + path = Path(path) + try: + ident = json.loads( + _sp.run( + ["mkvmerge", "-J", str(path)], stdin=_sp.DEVNULL, capture_output=True, text=True + ).stdout + ) + tracks = ident.get("tracks", []) + except Exception: + return False + if not tracks: + return False + cmd = ["mkvpropedit", str(path)] + for i, t in enumerate(tracks, start=1): + lang = t.get("properties", {}).get("language", "und") or "und" + cmd += ["--edit", f"track:@{i}", "--set", f"language-ietf={lang}"] + r = _sp.run(cmd, stdin=_sp.DEVNULL, stdout=_sp.DEVNULL, stderr=_sp.DEVNULL) + return r.returncode == 0 + + +def make_head_clip(src, out, seconds, run=None, cutter="mkvmerge"): + """Cut [0, seconds] from src into out (stream copy). Returns True on a non-empty output. + + The cutters have complementary side effects, so callers try each and validate: + - mkvmerge: preserves timestamp defects and und-language, but normalizes missing IETF language + tags (repairs the "Metadata errors" defect the large samples carry). + - ffmpeg (-bitexact): preserves the missing-IETF defect, but STRIPS IETF tags from clean files + (introducing a spurious SetLanguage) and can break DTS-repair clips. + - ffmpeg-tags (no -bitexact): preserves IETF on clean files AND the metadata defect where + present, at the cost of writing Lavf writing-app tags. + All add their own track tags (ffmpeg: DURATION even with -bitexact; mkvmerge: statistics) - + the caller strips them with mkvpropedit when the source had none. Non-mkv sources always use + ffmpeg (mkvmerge cannot write their containers).""" + import subprocess as _sp + + src, out = Path(src), Path(out) + for p in out.parent.glob(out.stem + ".*"): + p.unlink() + + def _run(cmd): + if run: + return run(cmd) + return _sp.run(cmd, stdin=_sp.DEVNULL, stdout=_sp.DEVNULL, stderr=_sp.DEVNULL) + + if cutter == "mkvmerge" and src.suffix.lower() == ".mkv": + _run(["mkvmerge", "-o", str(out), "--split", f"parts:00:00:00-{hms(seconds)}", str(src)]) + if not out.exists(): + numbered = out.with_name(out.stem + "-001" + out.suffix) + if numbered.exists(): + numbered.rename(out) + else: + cmd = [ + "ffmpeg", + "-hide_banner", + "-loglevel", + "error", + "-i", + str(src), + "-t", + str(int(seconds)), + "-map", + "0", + "-c", + "copy", + "-avoid_negative_ts", + "make_zero", + ] + if cutter != "ffmpeg-tags": + cmd.append("-bitexact") + _run(cmd + [str(out)]) + return out.exists() and out.stat().st_size > 0 + + +def is_filepath_quote(q): + return (q.startswith("/") and Path(q).suffix.lower() in (MEDIA_EXTS | {".tmp"})) or bool( + re.search(r"\.tmp\d+", q) + ) + + +def extract_errors(after): + """From the text after 'ExitCode: N', return error strings (excluding the appended filename).""" + if not after: + return set() + quoted = QUOTED_RE.findall(after) + if quoted: + payload = [q for q in quoted if not is_filepath_quote(q)] + else: + payload = [re.sub(r"\s*:\s*/\S+$", "", after).strip()] + errs = set() + for p in payload: + for e in p.split(" | "): + e = e.strip() + if e and not BENIGN_NOISE.search(e): + # normalize run-varying content so identical errors compare equal across runs: + # media-root paths (/Test/Media vs /media) and ASLR pointer addresses (0x...) + e = re.sub(r"(/Test/Media|/media)/", "/", e) + e = re.sub(r"0x[0-9a-fA-F]+", "0xADDR", e) + errs.add(e) + return errs + + +def parse_log(path): + """{stem: {detections:set, errors:set, tracks:list}} by thread affinity + line filename.""" + result = {} + thread_file = {} + path = Path(path) + + def bucket(stem): + return result.setdefault(stem, {"detections": set(), "errors": set(), "tracks": []}) + + if not path.exists(): + return result + for raw in path.read_text(errors="replace").splitlines(): + m = LINE_RE.match(raw) + if not m: + continue + tid, msg = m.group(1), m.group(2) + b = BEFORE_RE.search(msg) + if b: + thread_file[tid] = stem_of(b.group(1)) + continue + stem = thread_file.get(tid) + d = DETECT_RE.search(msg) + if d: + fn = re.search(r'"([^"]+\.[A-Za-z0-9]+)"\s*$', msg) + target = stem_of(fn.group(1)) if fn else stem + if target: + bucket(target)["detections"].add(d.group(1).strip()) + continue + t = TRACK_RE.search(msg) + if t and stem: + entry = {"type": t.group(1), "format": t.group(2), "interlaced": t.group(3) == "True"} + trk = bucket(stem)["tracks"] + if entry not in trk: + trk.append(entry) + continue + f = FAIL_RE.search(msg) + if f and stem: + errs = extract_errors(f.group(2)) + bucket(stem)["errors"] |= errs if errs else {""} + return result + + +def error_shape(e): + """Reduce an (already ADDR/path-normalized) error line to its PHYSICAL SHAPE: the exact + ffmpeg message template with site-varying content (stream indexes, MB coordinates, picture + numbers, sizes) normalized out. Distinct shapes are distinct physical error identities - + far finer than the broad signature classes, and stable across corruption sites and future + PlexCleaner logic refinements.""" + s = e + # decoder/stream context brackets -> keep only the codec identity + s = re.sub(r"\[[a-z]+#\d+:\d+/(\w+) @ 0xADDR\]\s*", r"[\1] ", s) # [vist#0:0/h264 @ ..] + s = re.sub(r"\[dec:(\w+) @ 0xADDR\]\s*", r"[\1] ", s) # [dec:h264 @ ..] + s = re.sub(r"\[(\w+) @ 0xADDR\]", r"[\1]", s) # [h264 @ ..] + s = re.sub(r"\[SWR @ 0xADDR\]", "[SWR]", s) + s = re.sub(r"stream \d+", "stream N", s) + s = re.sub(r"-?\b\d+(\.\d+)?\b", "N", s) # coordinates/ids/sizes + return re.sub(r"\s+", " ", s).strip() + + +def classify_signature(errors): + subs = set() + for e in errors: + low = e.lower() + for needle, label in DECODE_SIGNATURES: + if needle.lower() in low: + subs.add(label) + break + return subs + + +def buckets_for(state, detections, sig_subtypes, in_errors): + b = set() + for flag in state: + if flag in STATE_BUCKET: + b.add(STATE_BUCKET[flag]) + for det in detections: + for needle, label in DETECT_BUCKET: + if needle.lower() in det.lower(): + b.add(label) + b |= sig_subtypes + if in_errors: + b.add("Error") + return b + + +def find_run(explicit, channel="develop"): + if explicit: + p = Path(explicit) + return p if p.is_absolute() else REG_DIR / p + best, best_mt = None, 0 + for d in REG_DIR.glob("*/"): + log, bi = d / "PlexCleaner_process.log", d / "buildinfo.json" + if not (log.exists() and bi.exists()): + continue + try: + if json.loads(bi.read_text()).get("Channel") != channel: + continue + except Exception: + continue + if log.stat().st_mtime > best_mt: + best, best_mt = d, log.stat().st_mtime + if not best: + sys.exit(f"ERROR: no {channel} run under {REG_DIR}") + return best diff --git a/RegressionTests/locate_issue.py b/RegressionTests/locate_issue.py new file mode 100644 index 00000000..43835b96 --- /dev/null +++ b/RegressionTests/locate_issue.py @@ -0,0 +1,221 @@ +#!/usr/bin/env python3 +"""Find WHERE a localized decode signature lives, so a short issue-complete clip can be cut. + +Strategy (cheapest first): + +1. head-clip [0, W] (mkvmerge --split for mkv; ffmpeg -t copy otherwise -- reads only the head) +2. verify-decode the clip with the PlexCleaner image ffmpeg (matches the regression) and check + whether the file's catalog decode-signature substrings re-appear in stderr +3. if not reproduced in the head, a full-decode locate with -stats timestamp correlation reports + the approximate time of the first hit; ``--write-rules`` records a region window around it + +Fidelity: uses the image's ffmpeg, NOT host ffmpeg, because decode error messages differ by +version. The source corpus is READ-ONLY; clips live under scratch. +""" + +import argparse +import json +import os +import re +import subprocess +from pathlib import Path + +from corpus_common import DECODE_SIGNATURES, SRC_DIR, make_head_clip + +# Default paths for the reference server; override on the command line for another environment. +SCRATCH = Path("/data/media/PlexCleaner/scratch-trim") +IMAGE = "docker.io/ptr727/plexcleaner:develop" +WORK = SCRATCH / "locate" + +# map a catalog subtype label -> the stderr substrings that evidence it (for grep-back) +SUBTYPE_NEEDLES: dict[str, list[str]] = {} +for _needle, _label in DECODE_SIGNATURES: + SUBTYPE_NEEDLES.setdefault(_label, []).append(_needle) + +TIME_RE = re.compile(r"time=(\d+):(\d\d):(\d\d(?:\.\d+)?)") + + +def sh(cmd: list[str]) -> subprocess.CompletedProcess[str]: + return subprocess.run( + cmd, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + errors="replace", + check=False, + ) + + +# PlexCleaner's exact VerifyMedia args (no -map: default stream selection; -xerror aborts on first +# error). stats is -nostats for a plain reproduction check, -stats for timestamped locating. +def verify_args(input_path: str, stats: str = "-nostats") -> list[str]: + return [ + "-nostdin", + "-loglevel", + "error", + "-hide_banner", + stats, + "-abort_on", + "empty_output", + "-xerror", + "-fflags", + "+genpts", + "-analyzeduration", + "2G", + "-probesize", + "2G", + "-i", + input_path, + "-max_muxing_queue_size", + "1024", + "-f", + "null", + "-", + ] + + +def decode_stderr(media_dir: Path, name: str) -> str: + """Verify-decode name with the image ffmpeg using PlexCleaner's exact args; return stderr.""" + uid, gid = os.getuid(), os.getgid() + r = sh( + [ + "docker", + "run", + "--rm", + "--user", + f"{uid}:{gid}", + "--volume", + f"{media_dir}:/media:ro", + "--entrypoint", + "ffmpeg", + IMAGE, + ] + + verify_args(f"/media/{name}") + ) + return r.stdout + + +def reproduced(stderr: str, needles: list[str]) -> bool: + low = stderr.lower() + return any(n.lower() in low for n in needles) + + +def full_decode_locate(src: Path, needles: list[str]) -> float | None: + """Full-decode src (image ffmpeg + -stats, no -xerror); return the approx time (s) of the first + line matching any needle, using the nearest preceding 'time=' progress stamp. None if unseen.""" + uid, gid = os.getuid(), os.getgid() + args = [a for a in verify_args(f"/media/{src.name}", stats="-stats") if a != "-xerror"] + proc = subprocess.Popen( + [ + "docker", + "run", + "--rm", + "--user", + f"{uid}:{gid}", + "--volume", + f"{src.parent}:/media:ro", + "--entrypoint", + "ffmpeg", + IMAGE, + ] + + args, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + text=True, + errors="replace", + bufsize=1, + ) + last_t = 0.0 + hit: float | None = None + assert proc.stderr is not None + for raw in proc.stderr: + for chunk in raw.replace("\r", "\n").split("\n"): + tm = TIME_RE.search(chunk) + if tm: + last_t = int(tm.group(1)) * 3600 + int(tm.group(2)) * 60 + float(tm.group(3)) + if hit is None and any(n.lower() in chunk.lower() for n in needles): + hit = last_t + proc.wait() + return hit + + +def write_region(rules_path: Path, name: str, start: int, end: int, note: str) -> None: + """Merge a region window for name into the external rules file (create if absent).""" + data = json.loads(rules_path.read_text()) if rules_path.exists() else {} + data.setdefault("regions", {})[name] = {"start": start, "end": end, "note": note} + rules_path.write_text(json.dumps(data, indent=2, ensure_ascii=False)) + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--catalog", required=True, help="source catalog.json (from catalog_corpus.py)") + ap.add_argument("-w", "--window", type=int, default=60, help="head-clip seconds") + ap.add_argument("files", nargs="*", help="basenames (default: all decode-signature files)") + ap.add_argument( + "--full", + action="store_true", + help="full-decode locate: report the approx time of the first signature hit", + ) + ap.add_argument("--write-rules", help="with --full: write located region windows to this file") + ap.add_argument( + "--pad", + type=int, + default=40, + help="seconds of padding around a located time when writing a region", + ) + args = ap.parse_args() + + cat = json.loads(Path(args.catalog).read_text()) + WORK.mkdir(parents=True, exist_ok=True) + + if args.files: + targets = [e for e in cat["files"] if e["file"] in args.files] + else: + targets = [e for e in cat["files"] if e["decode_subtypes"]] + + if args.full: + rules_path = Path(args.write_rules) if args.write_rules else None + print(f"{'FILE':52} {'SUBTYPES':28} ERROR@") + for e in targets: + name = e["file"] + src = SRC_DIR / name + subs = e["decode_subtypes"] + needles = [n for s in subs for n in SUBTYPE_NEEDLES.get(s, [])] + if not src.exists(): + print(f"{name[:52]:52} {','.join(subs)[:28]:28} MISSING SOURCE") + continue + t = full_decode_locate(src, needles) + loc = f"~{t:.0f}s ({t / 60:.1f}m)" if t is not None else "NOT FOUND in full decode" + print(f"{name[:52]:52} {','.join(subs)[:28]:28} {loc}") + if rules_path is not None and t is not None: + start = max(0, int(t) - args.pad) + write_region( + rules_path, name, start, int(t) + args.pad, f"decode signature at ~{int(t)}s" + ) + return + + print(f"{'FILE':52} {'SUBTYPES':30} {'HEAD':>6} RESULT") + for e in targets: + name = e["file"] + src = SRC_DIR / name + subs = e["decode_subtypes"] + needles = [n for s in subs for n in SUBTYPE_NEEDLES.get(s, [])] + if not src.exists(): + print(f"{name[:52]:52} {','.join(subs)[:30]:30} {'-':>6} MISSING SOURCE") + continue + wd = WORK / Path(name).stem + if wd.exists(): + for p in wd.iterdir(): + p.unlink() + wd.mkdir(parents=True, exist_ok=True) + if not make_head_clip(src, wd / name, args.window): + print(f"{name[:52]:52} {','.join(subs)[:30]:30} {'-':>6} CLIP FAILED") + continue + ok = reproduced(decode_stderr(wd, name), needles) + verdict = f"head[{args.window}s] reproduces" if ok else f"not in first {args.window}s" + print(f"{name[:52]:52} {','.join(subs)[:30]:30} {'yes' if ok else 'no':>6} {verdict}") + + +if __name__ == "__main__": + main() diff --git a/RegressionTests/reduce_corpus.py b/RegressionTests/reduce_corpus.py new file mode 100644 index 00000000..c13df9be --- /dev/null +++ b/RegressionTests/reduce_corpus.py @@ -0,0 +1,380 @@ +#!/usr/bin/env python3 +"""Build the reduced (quickscan) corpus: shrink samples while PROVING every issue survives. + +Each candidate clip is processed through the PlexCleaner image and must match the source's +`catalog.json` entry (generated by ``catalog_corpus.py``) on ALL of: + +- State equality (the processing-decision fingerprint; catches issues with no log signature, + e.g. timestamp-only-DTS files whose verify never logs a failed-execution line) +- detections superset (every `` detected`` from the source re-surfaces) +- verify-error signatures superset (every captured error class re-surfaces) + +Any miss -> the original is kept whole so no issue is ever lost. + +Cutting strategy per file: + +- default: head-clip ``[0, --seconds]`` (global properties + defects known to live in the head) +- region overrides (from the external rules file, located by ``locate_issue.py``): issue-localized + windows for defects deep in the file +- the cutter ladder tries mkvmerge and ffmpeg cuts plus in-place IETF surgery, because the cutters + have mirror-image side effects and only the prove-equivalence gate can pick the safe one + +The source corpus is READ-ONLY. Clips, work dirs, and outputs live under scratch / --out. + +Media-specific region windows are NOT hard-coded here (that would embed private filenames); they +live in an external rules file next to the corpus. See ``reduction-rules.example.json``. + +Modes: + +- ``validate`` (default): cut + process each clip, compare to the catalog, report PASS/FAIL +- ``generate --out DIR``: same, but on PASS write the reduced file to DIR; on FAIL copy the + original whole. Writes a reduced ``catalog.json`` with per-file decision, sizes, and issue sets. +""" + +import argparse +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path + +from corpus_common import ( + SRC_DIR, + classify_signature, + hms, + make_head_clip, + parse_log, + set_language_ietf, + stem_of, + strip_language_ietf, +) + +# Default paths for the reference server; override on the command line for another environment. +SCRATCH = Path("/data/media/PlexCleaner/scratch-trim") +IMAGE = "docker.io/ptr727/plexcleaner:develop" +APP = "/PlexCleaner/Debug/PlexCleaner" + +Region = tuple[int, int] + + +def load_regions(path: Path) -> dict[str, Region]: + """Load issue-localized cut windows from the external rules file. + + The rules live WITH the media (never in source control, to keep private filenames out of the + tree). Absent file -> no regions (head-clip everything); see ``reduction-rules.example.json`` + for the schema and ``locate_issue.py`` for generating entries. + """ + if not path.exists(): + print( + f"No rules file at {path} - regions empty (head-clip only). " + f"See reduction-rules.example.json." + ) + return {} + data = json.loads(path.read_text()) + return {name: (int(r["start"]), int(r["end"])) for name, r in data.get("regions", {}).items()} + + +def make_clip( + src: Path, out: Path, seconds: int, regions: dict[str, Region], cutter: str = "mkvmerge" +) -> bool: + """Cut a clip: a region window if the file has one in the rules, else a head-clip.""" + region = regions.get(src.name) + if not region: + return make_head_clip(src, out, seconds) + start, end = region + for p in out.parent.glob(out.stem + ".*"): + p.unlink() + if cutter == "ffmpeg": + # ffmpeg region cut gives cleaner timestamps at the cut boundary (a mkvmerge region cut can + # add a spurious verify->repaired hiccup on remux-grade HEVC), at the usual IETF-strip cost + subprocess.run( + [ + "ffmpeg", + "-hide_banner", + "-loglevel", + "error", + "-ss", + str(start), + "-i", + str(src), + "-t", + str(end - start), + "-map", + "0", + "-c", + "copy", + "-avoid_negative_ts", + "make_zero", + "-bitexact", + str(out), + ], + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + else: + # mkvmerge region cut preserves timestamp defects (ffmpeg -copyts normalizes some away) + subprocess.run( + ["mkvmerge", "-o", str(out), "--split", f"parts:{hms(start)}-{hms(end)}", str(src)], + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + if not out.exists(): + numbered = out.with_name(out.stem + "-001" + out.suffix) + if numbered.exists(): + numbered.rename(out) + return out.exists() and out.stat().st_size > 0 + + +def process_clip(workdir: Path, settings_dir: Path) -> tuple[dict, dict[str, set[str]]]: + """Process ``workdir/media`` through the image; return (parsed log map, {stem: state set}). + + Log + results go to ``workdir/out``, a SEPARATE mount: PlexCleaner deletes unwanted non-media + files inside the media dir, so a log written there is deleted by the very run that wrote it. + """ + media, out = workdir / "media", workdir / "out" + out.mkdir(parents=True, exist_ok=True) + uid, gid = os.getuid(), os.getgid() + subprocess.run( + [ + "docker", + "run", + "--rm", + "--user", + f"{uid}:{gid}", + "--env", + "TZ=America/Los_Angeles", + "--volume", + f"{media}:/media:rw", + "--volume", + f"{out}:/out:rw", + "--volume", + f"{settings_dir}:/config:ro", + IMAGE, + APP, + "process", + "--settingsfile=/config/PlexCleaner.json", + "--logfile=/out/clip_process.log", + "--mediafiles=/media", + "--resultsfile=/out/clip_results.json", + "--parallel", + "--testsnippets", + ], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + logmap = parse_log(out / "clip_process.log") + states: dict[str, set[str]] = {} + try: + res = json.loads((out / "clip_results.json").read_text()) + for r in res["Results"]["Results"]: + states[stem_of(os.path.basename(r["OriginalFileName"]))] = { + s.strip() for s in (r.get("State") or "").split(",") if s.strip() + } + except Exception: + pass + return logmap, states + + +def human(n: float) -> str: + for unit in ("B", "K", "M", "G", "T"): + if n < 1024: + return f"{n:.0f}{unit}" + n /= 1024 + return f"{n:.0f}P" + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("sources", nargs="*", help="source basenames (default: all catalog files)") + ap.add_argument("-s", "--seconds", type=int, default=60, help="head-clip length") + ap.add_argument("--catalog", required=True, help="source catalog.json (from catalog_corpus.py)") + ap.add_argument("--rules", help="external region rules file (default: next to --catalog)") + ap.add_argument( + "--settings", + default="/data/media/PlexCleaner/RegressionTest", + help="dir containing the PlexCleaner.json to process clips with", + ) + ap.add_argument("--mode", choices=["validate", "generate"], default="validate") + ap.add_argument("--out", help="output dir for the reduced set (generate mode)") + args = ap.parse_args() + + if args.mode == "generate" and not args.out: + sys.exit("generate mode requires --out DIR") + + catalog_path = Path(args.catalog) + catalog = json.loads(catalog_path.read_text()) + entries = {e["file"]: e for e in catalog["files"]} + rules_path = Path(args.rules) if args.rules else catalog_path.parent / "reduction-rules.json" + regions = load_regions(rules_path) + print(f"Catalog : {args.catalog} ({len(entries)} files)") + print(f"Head window : {args.seconds}s regions: {len(regions)} mode={args.mode}\n") + + sources = args.sources or sorted(entries) + settings_dir = Path(args.settings) + work_root = SCRATCH / "work" + work_root.mkdir(parents=True, exist_ok=True) + out_dir = Path(args.out) if args.out else None + if out_dir: + out_dir.mkdir(parents=True, exist_ok=True) + + manifest = [] + passes = fails = kept = 0 + print(f"{'FILE':54} {'SRC':>7} {'CLIP':>7} {'RATIO':>6} RESULT") + for name in sources: + e = entries.get(name) + src = SRC_DIR / name + stem = stem_of(name) + short = stem[:53] + if e is None or not src.exists(): + miss = "CATALOG" if e is None else "SOURCE" + print(f"{short:54} {'-':>7} {'-':>7} {'-':>6} MISSING {miss}") + continue + gt_state = set(e["state"]) + gt_det = set(e["detections"]) + gt_subs = set(e["decode_subtypes"]) + ssz = src.stat().st_size + + # Cutter ladder: the cutters have mirror-image side effects, so try each and let the + # prove-equivalence gate pick the one that preserves this file's issues. Surgical rungs + # (in-place header edits, no remux): "*-noietf" re-injects the missing-IETF metadata defect + # a mkvmerge cut repairs; "*-fixietf" sets IETF on an ffmpeg cut so the clip enters verify + # metadata-clean and its timestamp defect drives the verify->repair chain. + if name in regions: + ladder = ["region", "region-noietf", "region-ffmpeg", "region-ffmpeg-fixietf"] + elif src.suffix.lower() == ".mkv": + ladder = ["mkvmerge", "mkvmerge-noietf", "ffmpeg-fixietf", "ffmpeg", "ffmpeg-tags"] + else: + ladder = ["ffmpeg"] + + ok = False + attempt: dict = {} + attempts: list[dict] = [] + for cutter in ladder: + fw = work_root / stem + if fw.exists(): + shutil.rmtree(fw) + (fw / "media").mkdir(parents=True) + # cut a PRISTINE clip outside the media dir (processing mutates/renames the media copy; + # the reduced corpus must ship the unprocessed clip with its issues intact) + pristine = fw / name + base = cutter.replace("-noietf", "").replace("-fixietf", "") + if base.startswith("region"): + made = make_clip( + src, + pristine, + args.seconds, + regions, + cutter="ffmpeg" if base == "region-ffmpeg" else "mkvmerge", + ) + else: + made = make_head_clip(src, pristine, args.seconds, cutter=base) + if made and cutter.endswith("-noietf"): + made = strip_language_ietf(pristine) + elif made and cutter.endswith("-fixietf"): + made = set_language_ietf(pristine) + if not made: + attempt = {"cutter": cutter, "error": "CLIP FAILED"} + continue + # Source close to / shorter than the window -> ship VERBATIM: a cut remuxes, silently + # repairing container/metadata defects, and already-short hand-made samples need no cut. + verbatim = pristine.stat().st_size >= 0.5 * ssz + if verbatim: + shutil.copy2(src, pristine) + elif pristine.suffix.lower() == ".mkv" and "ClearedTags" not in gt_state: + # both cutters add their own track tags (ffmpeg DURATION / mkvmerge statistics); + # when the source had none, strip them in place (no remux, defects untouched) so + # the clip does not pick up a spurious ClearedTags state + subprocess.run( + ["mkvpropedit", str(pristine), "--tags", "all:"], + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + shutil.copy2(pristine, fw / "media" / name) + + logmap, states = process_clip(fw, settings_dir) + cl = logmap.get(stem, {"detections": set(), "errors": set()}) + cl_state = states.get(stem, set()) + miss_d = gt_det - cl["detections"] + # error equivalence at the SIGNATURE-CLASS level: raw lines embed per-site coordinates + # (MB positions, picture numbers) and accumulate across every corrupt site in the + # source, which a single-site clip can never string-match + miss_e = gt_subs - classify_signature(cl["errors"]) + state_ok = cl_state == gt_state + ok = state_ok and not miss_d and not miss_e + attempt = { + "cutter": cutter, + "pristine": pristine, + "csz": pristine.stat().st_size, + "cl_state": cl_state, + "miss_d": miss_d, + "miss_e": miss_e, + "state_ok": state_ok, + "verbatim": verbatim, + } + attempts.append(attempt) + if ok or verbatim: + break # verbatim is cutter-independent; retrying cannot change it + + csz = attempt.get("csz", ssz) + ratio = f"{ssz / csz:.0f}x" + result = f"PASS ({attempt.get('cutter')})" if ok else "FAIL" + print(f"{short:54} {human(ssz):>7} {human(csz):>7} {ratio:>6} {result}") + if not ok and "error" in attempt: + print(f" {attempt['error']}") + elif not ok: + for a in attempts: # show every rung's failure so the per-cutter cause is visible + delta_plus = sorted(a["cl_state"] - gt_state) + delta_minus = sorted(gt_state - a["cl_state"]) + bits = [] + if delta_plus or delta_minus: + bits.append(f"state +{delta_plus} -{delta_minus}") + if a["miss_d"]: + bits.append(f"miss_det={sorted(a['miss_d'])}") + if a["miss_e"]: + bits.append(f"miss_sig={sorted(a['miss_e'])}") + print(f" [{a['cutter']}] {'; '.join(bits) or 'ok?'}") + + passes += ok + fails += not ok + if out_dir: + dest = out_dir / name + shutil.copy2(attempt["pristine"] if ok else src, dest) + kept += not ok + + manifest.append( + { + "file": name, + "decision": ("reduced" if ok else "kept-full") + if out_dir + else ("would-reduce" if ok else "would-keep"), + "method": ("verbatim" if attempt.get("verbatim") else attempt.get("cutter", "-")) + if ok + else "kept-full", + "source_bytes": ssz, + "clip_bytes": csz if ok else ssz, + "state_match": attempt.get("state_ok", False), + "clip_state": sorted(attempt.get("cl_state", set())), + "ground_state": sorted(gt_state), + "missing_detections": sorted(attempt.get("miss_d", set())), + "missing_signatures": sorted(attempt.get("miss_e", set())), + } + ) + + print(f"\nPASS={passes} FAIL={fails} kept-full={kept}") + mpath = (out_dir or SCRATCH) / "catalog.json" + mpath.write_text( + json.dumps( + {"schema": 1, "collection": "reduced", "file_count": len(manifest), "files": manifest}, + indent=2, + ensure_ascii=False, + ) + ) + print(f"catalog: {mpath}") + + +if __name__ == "__main__": + main() diff --git a/RegressionTests/reduction-rules.example.json b/RegressionTests/reduction-rules.example.json new file mode 100644 index 00000000..8c35d008 --- /dev/null +++ b/RegressionTests/reduction-rules.example.json @@ -0,0 +1,16 @@ +{ + "schema": 1, + "_comment": "Example media-specific reduction rules. The real file lives WITH the corpus (never in source control) so private filenames stay out of the repo. reduce_corpus.py reads the 'regions' map; locate_issue.py --write-rules generates entries by locating a decode signature in the source. Keys are source basenames as they appear in the corpus catalog.json; the names below are synthetic placeholders.", + "regions": { + "Example Show - S01E01.mkv": { + "start": 540, + "end": 620, + "note": "decode signature at ~560s, deep in the file (head-clip misses it)" + }, + "Example Movie (2020).mkv": { + "start": 1810, + "end": 1890, + "note": "interlaced-decode error near the 30m mark" + } + } +} From 7b4fe94873c421aec066433bb71894c21ce1880c Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Fri, 17 Jul 2026 08:56:38 -0700 Subject: [PATCH 02/12] Add Python Lint Config and CI Step for Regression Tooling Add ruff and mypy configuration for the RegressionTests Python utilities (mirroring the audited Financial-Modeling conventions), a CI lint step in the validate gate that runs them via uvx at pinned versions, matching VSCode lint tasks for local parity, and gitignore entries for the Python caches. This is the first Python in the repo; the tooling is stdlib-only and Python source follows the repo default CRLF line ending. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/validate-task.yml | 14 +++++++ .gitignore | 6 +++ .vscode/tasks.json | 59 ++++++++++++++++++++++++++++- RegressionTests/pyproject.toml | 22 +++++++++++ 4 files changed, 100 insertions(+), 1 deletion(-) create mode 100644 RegressionTests/pyproject.toml diff --git a/.github/workflows/validate-task.yml b/.github/workflows/validate-task.yml index a8dbd40d..396bc0f5 100644 --- a/.github/workflows/validate-task.yml +++ b/.github/workflows/validate-task.yml @@ -84,6 +84,20 @@ jobs: HISTORY.md incremental_files_only: false + # Lint the RegressionTests Python tooling with ruff + mypy, pinned to the same versions the + # editor tasks run, via uvx (no project install; the tooling is stdlib-only). Config lives in + # RegressionTests/pyproject.toml, so run from that directory (mypy resolves config from CWD). + - name: Setup uv step + uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 + + - name: Lint Python step + working-directory: RegressionTests + run: | + set -euo pipefail + uvx ruff@0.15.22 check . + uvx ruff@0.15.22 format --check . + uvx mypy@2.3.0 . + - name: Lint workflows step uses: raven-actions/actionlint@3d39aea434753780c3b3d4a1a31c854b4dbf49d7 # v2.2.0 diff --git a/.gitignore b/.gitignore index 21241c2f..d0835d02 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,9 @@ coverage/ *.log *.user + +# Python tooling (RegressionTests/) +__pycache__/ +*.pyc +.mypy_cache/ +.ruff_cache/ diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 63c9bb8d..0c09f1cc 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -323,6 +323,60 @@ "clear": false } }, + { + "label": "Lint: Ruff", + "type": "shell", + "command": "uvx", + "args": [ + "ruff@0.15.22", + "check", + "." + ], + "options": { + "cwd": "${workspaceFolder}/RegressionTests" + }, + "problemMatcher": [], + "presentation": { + "showReuseMessage": false, + "clear": false + } + }, + { + "label": "Lint: Ruff Format", + "type": "shell", + "command": "uvx", + "args": [ + "ruff@0.15.22", + "format", + "--check", + "." + ], + "options": { + "cwd": "${workspaceFolder}/RegressionTests" + }, + "problemMatcher": [], + "presentation": { + "showReuseMessage": false, + "clear": false + } + }, + { + "label": "Lint: Mypy", + "type": "shell", + "command": "uvx", + "args": [ + "mypy@2.3.0", + "." + ], + "options": { + "cwd": "${workspaceFolder}/RegressionTests" + }, + "problemMatcher": [], + "presentation": { + "showReuseMessage": false, + "clear": false + } + }, { "label": "Lint: All (CI parity)", "dependsOrder": "sequence", @@ -332,7 +386,10 @@ "Lint: EditorConfig", "Lint: Workflows", "Lint: Markdown", - "Lint: Spelling" + "Lint: Spelling", + "Lint: Ruff", + "Lint: Ruff Format", + "Lint: Mypy" ], "problemMatcher": [], "presentation": { diff --git a/RegressionTests/pyproject.toml b/RegressionTests/pyproject.toml new file mode 100644 index 00000000..b778f9fd --- /dev/null +++ b/RegressionTests/pyproject.toml @@ -0,0 +1,22 @@ +# Linter-only configuration for the RegressionTests Python utilities. +# +# These are standalone stdlib-only scripts (no runtime dependencies), so this file carries no +# project/build metadata - only the ruff + mypy config, mirroring the ptr727 Financial-Modeling +# conventions. Run the tools with uv (no install needed): +# +# uvx ruff check . +# uvx ruff format --check . +# uvx mypy . + +[tool.ruff] +target-version = "py313" +line-length = 100 + +[tool.ruff.lint] +extend-select = ["I"] # import sorting (isort) + +[tool.mypy] +python_version = "3.13" +warn_unused_ignores = true +warn_redundant_casts = true +no_implicit_optional = true From 3c39b25e8003a74bfd151527b032ded3631057b8 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Fri, 17 Jul 2026 08:56:57 -0700 Subject: [PATCH 03/12] Add Regression Tests README Document the harness usage, the full-file-scan rationale, the tooling, the external rules format and generate-on-demand flow, the cutter ladder and surgical IETF rungs, naming conventions, the update-validate-snapshot loop, and the physical-shape coverage audit. All media titles are generalized. Co-Authored-By: Claude Opus 4.8 --- RegressionTests/README.md | 182 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 182 insertions(+) create mode 100644 RegressionTests/README.md diff --git a/RegressionTests/README.md b/RegressionTests/README.md new file mode 100644 index 00000000..d0fac701 --- /dev/null +++ b/RegressionTests/README.md @@ -0,0 +1,182 @@ +# Regression Tests + +Tooling and a reproducible process for verifying that PlexCleaner's processing decisions stay +consistent across versions, using a curated collection of troublesome media files. + +PlexCleaner's behavior depends heavily on the specific media it processes: most functional changes +are driven by a media file or media tool quirk affecting playback. This suite pins that behavior by +processing the same collection through successive builds and comparing the results down to the +per-file processing decision. + +## What is (and is not) in this directory + +The committed contents are code plus a synthetic example. No media and no media filenames live in +the repository: + +- The **media collection** lives on a server, next to the media (like a Plex library would). It is + never committed. The copyrighted filenames stay out of source control entirely. +- The **media-specific reduction rules** (issue-localized cut windows) also live with the media, in + an external JSON file the tooling reads and writes. The repo ships only + [`reduction-rules.example.json`](reduction-rules.example.json) with synthetic placeholder names. + +## Layout + +- [`RegressionTest.sh`](RegressionTest.sh) -- the harness: provision a test dataset, process it + through one Docker image tag, and write per-version results and logs for diffing. +- [`corpus_common.py`](corpus_common.py) -- shared library: log parsing, deterministic issue + classification, and the clip and metadata-surgery helpers. +- [`catalog_corpus.py`](catalog_corpus.py) -- derive a machine-readable issue catalog + (`catalog.json`) for a collection from a processing run. +- [`reduce_corpus.py`](reduce_corpus.py) -- build and validate a reduced collection: shrink each + sample while proving every issue survives. +- [`locate_issue.py`](locate_issue.py) -- find where a decode signature lives in a file, and + optionally record the located window into the external rules file. +- [`audit_physical.py`](audit_physical.py) -- physical-error-shape coverage audit of the reduced + collection. +- [`reduction-rules.example.json`](reduction-rules.example.json) -- synthetic example of the + external rules schema. +- [`pyproject.toml`](pyproject.toml) -- ruff and mypy configuration for the Python tooling. + +## The collection + +Two collections live side by side in the media dataset, each with its own generated `catalog.json`: + +- `full/` -- the complete troublesome samples. The source of truth. +- `reduced/` -- shorter clips derived from `full/`, each proven to reproduce the same issue set. A + reduced run is far faster (minutes instead of hours) and is the default for iterating. + +Both are read-only during a run. The harness never mutates the collection; it processes a +disposable copy. + +## Running the harness + +`RegressionTest.sh` provisions a disposable test dataset as a zero-copy ZFS clone of the newest +collection snapshot, processes it through a single Docker image tag, and writes results into a +directory named after the image build version so runs stay durable and version-to-version +comparable. + +```shell +sudo ./RegressionTest.sh [quick|full] [tag] [corpus] [plugin...] +``` + +- `quick` (default) keeps full-file scanning as in production but writes short test snippets to + shorten remux and re-encode. `full` also processes the complete media. +- `tag` is the Docker image tag to test (default `develop`). +- `corpus` selects `full` (default) or `reduced`. +- `plugin` names optional example plugins to build and run after processing, to confirm a plugin + loads and runs against the processed dataset. + +Provisioning uses ZFS clones rather than `rsync`: the clone is instant and drift-free, and a +rollback to the clone's own snapshot works even while long-running media containers hold the mount. + +### Full-file scanning + +The harness scans the whole file; it does not use `--quickscan`. Bounding the scan to the start of +the file causes false negatives for defects that surface later, notably closed-caption detection +and interlace detection, so a full scan is the correct default for regression comparison. + +## The issue catalog + +`catalog_corpus.py` turns a processing run into `catalog.json`: one entry per file recording its +processing State, the detections it triggered, and the classified decode-error subtypes. This +catalog is the ground truth the reduction proves against, and the artifact compared between +versions. + +Classification is deterministic and lives in `corpus_common.py`: raw ffmpeg error lines carry +per-site coordinates (macroblock positions, picture numbers) and accumulate across every corrupt +site, so they are normalized to a stable signature class before comparison. + +## Reducing the collection + +`reduce_corpus.py` shrinks each sample while proving no issue is lost. A candidate clip is processed +through the image and must match the source catalog entry on all of: + +- State equality (the processing-decision fingerprint, which catches issues that leave no log + signature). +- detections superset (every detection re-surfaces). +- verify-error signatures superset (every error class re-surfaces). + +Any miss keeps the original whole, so an issue is never dropped. + +### Cutter ladder + +Cutting a clip can silently repair the very defect the sample exists to capture, and the two cutters +have mirror-image side effects: an `mkvmerge` cut preserves timestamp defects but strips +language-IETF metadata, while an `ffmpeg` cut preserves metadata but normalizes some timestamp +defects. So the tool tries a ladder of cutters plus in-place metadata surgery and lets the +prove-equivalence gate pick the one that keeps this file's issues: + +- head clips and region clips via `mkvmerge` and `ffmpeg`. +- surgical rungs that edit the header in place with no remux: a `noietf` rung re-injects the + missing-IETF-metadata defect an `mkvmerge` cut would repair, and a `fixietf` rung sets IETF on an + `ffmpeg` cut so a timestamp defect drives the verify-and-repair chain. + +Samples at or near the window length ship verbatim, because any cut remuxes and would repair +container or metadata defects. + +### Region rules (generate on demand) + +Most defects live in the head of the file, so the default is a head clip. Defects deep in a file +need an issue-localized window. Those windows are media-specific, so they are not hard-coded; they +live in an external rules file next to the collection (default `reduction-rules.json` beside the +catalog). + +Generate them from your own media on demand: + +```shell +# locate the decode signature and record a padded window into the rules file +python3 locate_issue.py --catalog /path/to/full/catalog.json --full --write-rules /path/to/full/reduction-rules.json + +# build the reduced collection, reading those windows +python3 reduce_corpus.py --catalog /path/to/full/catalog.json --mode generate --out /path/to/reduced +``` + +The rules schema is a `regions` map keyed by source basename; see +[`reduction-rules.example.json`](reduction-rules.example.json). If the rules file is absent, every +file is head-clipped and the tool says so. + +## Physical-shape coverage audit + +The reduction gate compares broad signature classes, and a class can lump several distinct physical +ffmpeg messages together. `audit_physical.py` closes that gap: it extracts every physical error +shape (the exact message template, with run- and site-varying content normalized out) from the +ground run and from each reduced clip, and reports any source shape a clip fails to reproduce. It +augments the reduced `catalog.json` with per-file and corpus-level coverage figures, so an +under-covered area is always visible and it is known when a change warrants a full-collection run. + +## Naming conventions + +Collection filenames follow a small set of conventions so the catalog stays readable: + +- a descriptive real title for a naturally occurring sample. +- a codec-matrix name (`codec_container`) for a sample that exists to exercise a specific + combination. +- a `Word-Word` behavior name for a sample built to test one behavior. +- a `[container]` disambiguation tag appended only when two samples would otherwise collide on the + output stem (PlexCleaner renames every output to `.mkv`). +- a filename fixture whose media is a tiny synthetic clip and whose filename is the actual test. + +## Update, validate, snapshot + +The working loop when the collection changes: + +1. Update the collection (add or adjust a sample). +2. Regenerate the affected catalog with `catalog_corpus.py`. +3. Rebuild and validate the reduced collection with `reduce_corpus.py`, and audit coverage with + `audit_physical.py`. +4. Snapshot the dataset so a run can clone from it. + +## Python tooling + +The Python utilities are standalone and stdlib-only (subprocess, json, argparse, pathlib, re). They +are linted with ruff and type-checked with mypy; the configuration is in +[`pyproject.toml`](pyproject.toml). Run them via `uvx`, which needs no install: + +```shell +uvx ruff check . +uvx ruff format --check . +uvx mypy . +``` + +The same commands run in CI and are available as VSCode tasks. Python source is CRLF, matching the +repository's default line-ending convention. From 29320358cde4b05f4ea14f782d278a1604af955b Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Fri, 17 Jul 2026 08:57:20 -0700 Subject: [PATCH 04/12] Document the Regression Test Suite in the Repo Docs Replace the superseded ad-hoc regression section in the README with a pointer to RegressionTests/, add the directory and a Python line-ending note to AGENTS.md, add a Testing pointer in ARCHITECTURE.md, and record the addition in HISTORY.md under the unreleased 3.21 version. Co-Authored-By: Claude Opus 4.8 --- AGENTS.md | 2 ++ ARCHITECTURE.md | 4 ++++ HISTORY.md | 1 + README.md | 64 ++----------------------------------------------- 4 files changed, 9 insertions(+), 62 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 82f6bad6..9e1cee39 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -106,6 +106,7 @@ Applies to code and workflow (`#`) comments alike. - [`.editorconfig`](./.editorconfig) is the single source of truth for line endings: CRLF for `.md`, `.cs`, XML/`.csproj`/`.props`, non-workflow `.yml`/`.yaml`, `.json`, `.cmd`/`.bat`/`.ps1`; LF for `.sh`, Dockerfiles, and workflow YAML (`.github/workflows/*.{yml,yaml}`). Workflow YAML is pinned LF because Dependabot and Actions rewrite it with LF, so declaring LF keeps it consistent instead of mixed; git still leaves endings alone (`* -text`) and CI's `editorconfig-checker` enforces it. The `[*.cs]`/ReSharper style block applies because this repo ships .NET. - **Always honor the `.editorconfig` ending.** Create a file with its spec ending; when editing a file, bring the whole file to spec (a file-wide EOL fix alongside the content change is expected, not a violation); if you come across a file with the wrong ending, fix it. [`.gitattributes`](./.gitattributes) (`* -text`) governs git's own normalization - it is not a license to leave a file on the wrong ending. Verify with `file ` after writing. +- **Python (`.py`) and `.toml` are CRLF.** They have no `[*.py]`/`[*.toml]` override, so they inherit the `[*]` CRLF default (matching the audited convention that keeps Python on the repo default rather than pinning LF). Only the `.sh` harness is LF. ### Quantitative Claims @@ -221,6 +222,7 @@ An **expected, recoverable** failure escalates through the standard repair tiers - **PlexCleanerTests** (`PlexCleanerTests/PlexCleanerTests.csproj`) - xUnit v3 test suite. Assertions via AwesomeAssertions. - **`Docker/`** - multi-arch Linux container build (`ubuntu:rolling`, `linux/amd64` + `linux/arm64`); runs as a `nonroot` user, mounts media under `/media`. +- **`RegressionTests/`** - regression harness and tooling: a ZFS-clone Bash harness plus standalone stdlib-only Python utilities (catalog / reduce / locate / audit) that verify processing decisions stay consistent across versions against a curated media collection. The Python tooling is linted with ruff and type-checked with mypy (config in `RegressionTests/pyproject.toml`); it is the only Python in the repo. No media or media filenames are committed - media-specific reduction rules live with the media as an external JSON file, and the repo ships only a synthetic example. See [`RegressionTests/README.md`](./RegressionTests/README.md). - **Build configuration**: - Common MSBuild properties (`TargetFramework`, `Nullable`, `ImplicitUsings`, `AnalysisLevel`, etc.) live in `Directory.Build.props` at the solution root. Do not duplicate these in individual `.csproj` files - only add a property to a `.csproj` when it is project-specific or overrides the shared default. - All NuGet package versions are centralised in `Directory.Packages.props`. `PackageReference` elements in `.csproj` files must not include a `Version` attribute. Asset metadata (`PrivateAssets`, `IncludeAssets`) stays in the `.csproj` `PackageReference` element. diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index ef3edc61..49451373 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -316,6 +316,10 @@ Check states with `HasFlag()`, combine with `|=` - Docker tests: Download Matroska test files from GitHub - CI: Separate workflows for build tests and Docker tests +### Regression Testing + +- Cross-version processing-consistency checks against a curated media collection, with a ZFS-clone harness and Python catalog / reduce / locate / audit tooling under `RegressionTests/`. See [`RegressionTests/README.md`](./RegressionTests/README.md). + ## Build and Release The authoritative release and workflow governance is in [AGENTS.md](./AGENTS.md). This section is a short architectural summary. diff --git a/HISTORY.md b/HISTORY.md index 9674a8bd..2d4f954a 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -42,6 +42,7 @@ Utility to optimize media files for Direct Play in Plex, Emby, Jellyfin, etc. - Handle the `SIGINT`, `SIGTERM`, and `SIGQUIT` termination signals (`docker stop`, `Ctrl+C`) so processing is interrupted gracefully and the summary and exit code are logged before exit. The custom `Ctrl+Q`/`Ctrl+Z` exit keys are removed in favor of the standard signals. - Normalize `Default` track flags instead of only warning about them: clear the flag on a lone track of a type, keep the preferred audio track as the single default when multiple are flagged, and clear all default flags on subtitle tracks. - Added a `custom` command that loads a user-provided plugin assembly implementing `IProcessPlugin` and runs it over the media files, reusing the file iteration and processing API for bespoke re-processing or repair. Includes the `MatroskaHeaderCleanup` example plugin. Not available in AOT builds. + - Added a regression test suite and reduced-corpus tooling under `RegressionTests/`: a ZFS-clone harness and Python catalog / reduce / locate / audit utilities that verify processing decisions stay consistent across versions. No application changes. - Version 3.19: - Reworked the CI/CD pipeline to a branch-scoped self-publishing model: a weekly scheduled run (and manual dispatch) publishes both `main` (stable, Docker `latest`) and `develop` (prerelease, Docker `develop`) - native executables, the multi-arch Docker image, and the GitHub release - while merges accumulate until the next run. No application changes. - Added `WORKFLOW.md` (the canonical CI/CD specification) and `repo-config/` (rulesets and repository settings as code). diff --git a/README.md b/README.md index 1fa5b36b..159c6e78 100644 --- a/README.md +++ b/README.md @@ -936,69 +936,9 @@ docker run \ ### Regression Testing -Regression testing ensures consistent behavior across versions by comparing processing results on the same media files. +Regression testing ensures consistent behavior across versions by comparing processing results on the same media files, down to the per-file processing decision. -The behavior of the tool is very dependent on the media files being tested, and the following process can facilitate regressions testing, assuring that the process results between versions remain consistent. - -- Maintain a collection of troublesome media files that resulted in functional changes. -- Create a ZFS snapshot of the media files to test. -- Process the files, using a known good version, and save the results in JSON format using the `--resultsfile` option. -- Restore the ZFS snapshot allowing repetitive testing using the original files. -- Process the files again using the under test version. -- Compare the JSON results file from the known good version with the version under test. -- Investigate any file comparison discrepancies. - -E.g. - -```shell -# Copy troublesome files -rsync -av --delete --progress /data/media/Troublesome/. /data/media/test -chown -R nobody:users /data/media/test -chmod -R ug=rwx,o=rx /data/media/test - -# Take snapshot -zfs destroy hddpool/media/test@backup -zfs snapshot hddpool/media/test@backup -``` - -```shell -# Config -PlexCleanerApp=/PlexCleaner/Debug/PlexCleaner -MediaPath=/Test/Media -ConfigPath=/Test/Config - -# Test function -RunContainer () { - local Image=$1 - local Tag=$2 - - # Rollback to snapshot - sudo zfs rollback hddpool/media/test@backup - - # Process files - docker run \ - -it \ - --rm \ - --pull always \ - --name PlexCleaner-Test \ - --user nobody:users \ - --env TZ=America/Los_Angeles \ - --volume /data/media/test:$MediaPath:rw \ - --volume /data/media/PlexCleaner:$ConfigPath:rw \ - $Image:$Tag \ - $PlexCleanerApp process \ - --settingsfile=$ConfigPath/PlexCleaner.json \ - --logfile=$ConfigPath/PlexCleaner-$Tag.log \ - --mediafiles=$MediaPath \ - --testsnippets \ - --quickscan \ - --resultsfile=$ConfigPath/Results-$Tag.json -} - -# Test containers -RunContainer docker.io/ptr727/plexcleaner latest -RunContainer docker.io/ptr727/plexcleaner develop -``` +The behavior of the tool is very dependent on the media files being tested. A reproducible process and its tooling live under [`RegressionTests/`](./RegressionTests/): a ZFS-clone harness that processes a curated collection of troublesome media through a given image tag, plus utilities that derive a machine-readable issue catalog, build a proven-equivalent reduced collection, and audit physical-error coverage. See [`RegressionTests/README.md`](./RegressionTests/README.md) for details. ## Development Tooling From 55db716c353375b973f371f56670e0ec69d1bff9 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Fri, 17 Jul 2026 08:59:07 -0700 Subject: [PATCH 05/12] Recommend the Python Extensions for the Regression Tooling Add the ruff, mypy, Pylance, and Python extension recommendations so the editor surfaces the RegressionTests lint and type-check tooling. Recommendations only; no Python workspace settings are needed for the stdlib-only scripts. Co-Authored-By: Claude Opus 4.8 --- PlexCleaner.code-workspace | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/PlexCleaner.code-workspace b/PlexCleaner.code-workspace index b036bfe8..1644cde7 100644 --- a/PlexCleaner.code-workspace +++ b/PlexCleaner.code-workspace @@ -32,15 +32,19 @@ }, "extensions": { "recommendations": [ + "charliermarsh.ruff", "csharpier.csharpier-vscode", "davidanson.vscode-markdownlint", "editorconfig.editorconfig", + "fanaticpythoner.better-todo-tree", "github.vscode-github-actions", "ms-azuretools.vscode-docker", "ms-dotnettools.csdevkit", + "ms-python.mypy-type-checker", + "ms-python.python", + "ms-python.vscode-pylance", "streetsidesoftware.code-spell-checker", - "yzhang.markdown-all-in-one", - "fanaticpythoner.better-todo-tree" + "yzhang.markdown-all-in-one" ] } } From 783fab4ed9fcd6ad5f5ddef441664e9be75b8f55 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Fri, 17 Jul 2026 09:06:39 -0700 Subject: [PATCH 06/12] Harden the Reduce Gate and Drop Redundant Sudo Address Copilot review feedback: - reduce_corpus.py: process_clip now returns a None State map when a run does not complete (missing or unreadable results file) and the caller treats that rung as failed, so a failed run can no longer masquerade as an empty-State PASS and weaken the equivalence gate. - RegressionTest.sh: drop sudo from the zfs calls; the script already exits unless run as uid 0, so sudo was redundant and would break on a root host without sudo installed. Co-Authored-By: Claude Opus 4.8 --- RegressionTests/RegressionTest.sh | 12 ++++++------ RegressionTests/reduce_corpus.py | 30 +++++++++++++++++++++--------- 2 files changed, 27 insertions(+), 15 deletions(-) diff --git a/RegressionTests/RegressionTest.sh b/RegressionTests/RegressionTest.sh index 25f8b6e2..ebcfc08a 100755 --- a/RegressionTests/RegressionTest.sh +++ b/RegressionTests/RegressionTest.sh @@ -122,21 +122,21 @@ ProvisionDataset() { Origin="$(zfs get -H -o value origin "$TestDataset" 2>/dev/null || true)" if [[ "$Origin" == "$CorpusSnap" ]]; then echo "Test clone current ($CorpusSnap) - rolling back" - sudo zfs rollback "$Snapshot" + zfs rollback "$Snapshot" return fi echo "Provisioning $TestDataset as a clone of $CorpusSnap" if zfs list "$TestDataset" >/dev/null 2>&1; then - sudo zfs rename "$TestDataset" "$TestDataset-retired-$(date +%s)" + zfs rename "$TestDataset" "$TestDataset-retired-$(date +%s)" fi - sudo zfs clone "$CorpusSnap" "$TestDataset" - sudo zfs snapshot "$Snapshot" + zfs clone "$CorpusSnap" "$TestDataset" + zfs snapshot "$Snapshot" # best-effort cleanup of retired clones (succeeds once the holders have restarted/closed) local Retired for Retired in $(zfs list -H -o name 2>/dev/null | grep -E "^$TestDataset-retired-" || true); do - sudo zfs destroy -r "$Retired" 2>/dev/null && + zfs destroy -r "$Retired" 2>/dev/null && echo "Destroyed retired clone $Retired" || echo "Retired clone $Retired still held; will retry next run" >&2 done @@ -145,7 +145,7 @@ ProvisionDataset() { # Restore the test dataset (clone) to its pristine post-provision state RestoreDataset() { echo "Restoring test dataset" - sudo zfs rollback "$Snapshot" + zfs rollback "$Snapshot" } # GetImageVersion Tag -> prints the build version from the image "version" label diff --git a/RegressionTests/reduce_corpus.py b/RegressionTests/reduce_corpus.py index c13df9be..fe1797aa 100644 --- a/RegressionTests/reduce_corpus.py +++ b/RegressionTests/reduce_corpus.py @@ -128,11 +128,15 @@ def make_clip( return out.exists() and out.stat().st_size > 0 -def process_clip(workdir: Path, settings_dir: Path) -> tuple[dict, dict[str, set[str]]]: +def process_clip(workdir: Path, settings_dir: Path) -> tuple[dict, dict[str, set[str]] | None]: """Process ``workdir/media`` through the image; return (parsed log map, {stem: state set}). Log + results go to ``workdir/out``, a SEPARATE mount: PlexCleaner deletes unwanted non-media files inside the media dir, so a log written there is deleted by the very run that wrote it. + + The State map is ``None`` when the run did not complete (missing or unreadable results file), + so a failed run cannot be mistaken for an empty-State PASS and silently weaken the gate. A + completed run always writes ``clip_results.json``, even when a file's State is empty. """ media, out = workdir / "media", workdir / "out" out.mkdir(parents=True, exist_ok=True) @@ -166,15 +170,19 @@ def process_clip(workdir: Path, settings_dir: Path) -> tuple[dict, dict[str, set stderr=subprocess.DEVNULL, ) logmap = parse_log(out / "clip_process.log") - states: dict[str, set[str]] = {} + results_file = out / "clip_results.json" + if not results_file.exists(): + return logmap, None # run did not complete: not an empty result, a failure try: - res = json.loads((out / "clip_results.json").read_text()) - for r in res["Results"]["Results"]: - states[stem_of(os.path.basename(r["OriginalFileName"]))] = { - s.strip() for s in (r.get("State") or "").split(",") if s.strip() - } - except Exception: - pass + res = json.loads(results_file.read_text()) + results = res["Results"]["Results"] + except (json.JSONDecodeError, KeyError, OSError): + return logmap, None + states: dict[str, set[str]] = {} + for r in results: + states[stem_of(os.path.basename(r["OriginalFileName"]))] = { + s.strip() for s in (r.get("State") or "").split(",") if s.strip() + } return logmap, states @@ -296,6 +304,10 @@ def main() -> None: shutil.copy2(pristine, fw / "media" / name) logmap, states = process_clip(fw, settings_dir) + if states is None: + # processing did not complete: a failed run must not pass as an empty result + attempt = {"cutter": cutter, "error": "PROCESS FAILED"} + continue cl = logmap.get(stem, {"detections": set(), "errors": set()}) cl_state = states.get(stem, set()) miss_d = gt_det - cl["detections"] From dcaf333facfd12dddf79170f9d96d044ecf149d1 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Fri, 17 Jul 2026 09:09:09 -0700 Subject: [PATCH 07/12] Drop the Non-Functional Shebangs From the Python Tooling Address Copilot review feedback: the utilities are CRLF (the audited repo convention) and are invoked via python3 or uvx, never executed directly, and they carry no executable bit. A shebang on a CRLF file is non-functional and misleading (direct execution would fail on the CR), so remove it. Co-Authored-By: Claude Opus 4.8 --- RegressionTests/audit_physical.py | 1 - RegressionTests/catalog_corpus.py | 1 - RegressionTests/locate_issue.py | 1 - RegressionTests/reduce_corpus.py | 1 - 4 files changed, 4 deletions(-) diff --git a/RegressionTests/audit_physical.py b/RegressionTests/audit_physical.py index 42b5cfee..2d10abcb 100644 --- a/RegressionTests/audit_physical.py +++ b/RegressionTests/audit_physical.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """ audit_physical.py - physical-error-shape audit of the reduced corpus. diff --git a/RegressionTests/catalog_corpus.py b/RegressionTests/catalog_corpus.py index 6fee131c..15c89a4f 100644 --- a/RegressionTests/catalog_corpus.py +++ b/RegressionTests/catalog_corpus.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """ catalog_corpus.py - derive a reproducible, machine-readable issue catalog for a media corpus from a versioned PlexCleaner regression run. diff --git a/RegressionTests/locate_issue.py b/RegressionTests/locate_issue.py index 43835b96..b07e7015 100644 --- a/RegressionTests/locate_issue.py +++ b/RegressionTests/locate_issue.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """Find WHERE a localized decode signature lives, so a short issue-complete clip can be cut. Strategy (cheapest first): diff --git a/RegressionTests/reduce_corpus.py b/RegressionTests/reduce_corpus.py index fe1797aa..a1ef40db 100644 --- a/RegressionTests/reduce_corpus.py +++ b/RegressionTests/reduce_corpus.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """Build the reduced (quickscan) corpus: shrink samples while PROVING every issue survives. Each candidate clip is processed through the PlexCleaner image and must match the source's From 756691c1232e6e95444a9cb0b6d945101686d41c Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Fri, 17 Jul 2026 09:14:05 -0700 Subject: [PATCH 08/12] Chown Results Dir to the Container User and Pin Doc Commands Address Copilot review feedback: - RegressionTest.sh: the version results directory is populated as root but the containers run as nobody:users and write logs and results into it; chown it to that user before the container runs so the writes succeed on any host, independent of the parent directory's ownership or ACLs. - pyproject.toml and README.md: pin the ruff and mypy versions in the example commands to match CI and the VSCode tasks, so local runs cannot diverge from CI on an unpinned newer tool. Co-Authored-By: Claude Opus 4.8 --- RegressionTests/README.md | 11 ++++++----- RegressionTests/RegressionTest.sh | 5 +++++ RegressionTests/pyproject.toml | 8 +++++--- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/RegressionTests/README.md b/RegressionTests/README.md index d0fac701..dffca9e2 100644 --- a/RegressionTests/README.md +++ b/RegressionTests/README.md @@ -173,10 +173,11 @@ are linted with ruff and type-checked with mypy; the configuration is in [`pyproject.toml`](pyproject.toml). Run them via `uvx`, which needs no install: ```shell -uvx ruff check . -uvx ruff format --check . -uvx mypy . +uvx ruff@0.15.22 check . +uvx ruff@0.15.22 format --check . +uvx mypy@2.3.0 . ``` -The same commands run in CI and are available as VSCode tasks. Python source is CRLF, matching the -repository's default line-ending convention. +The versions are pinned so local runs match CI exactly; the same pinned commands run in CI and are +available as VSCode tasks. Python source is CRLF, matching the repository's default line-ending +convention. diff --git a/RegressionTests/RegressionTest.sh b/RegressionTests/RegressionTest.sh index ebcfc08a..b339cd38 100755 --- a/RegressionTests/RegressionTest.sh +++ b/RegressionTests/RegressionTest.sh @@ -349,6 +349,11 @@ RunRegressionTests() { fi done + # The version directory was populated as root (mkdir, settings copy, buildinfo, plugin DLLs), but + # the containers run as nobody:users and write their logs and results into it; hand it to that + # user so the writes succeed regardless of the parent directory's ownership or ACLs. + chown -R nobody:users "$VersionDir" + RunDefaultSettings "$Tag" "$Version" # RunCreateSchema "$Tag" "$Version" diff --git a/RegressionTests/pyproject.toml b/RegressionTests/pyproject.toml index b778f9fd..e5d3b4dc 100644 --- a/RegressionTests/pyproject.toml +++ b/RegressionTests/pyproject.toml @@ -4,9 +4,11 @@ # project/build metadata - only the ruff + mypy config, mirroring the ptr727 Financial-Modeling # conventions. Run the tools with uv (no install needed): # -# uvx ruff check . -# uvx ruff format --check . -# uvx mypy . +# uvx ruff@0.15.22 check . +# uvx ruff@0.15.22 format --check . +# uvx mypy@2.3.0 . +# +# The versions are pinned to match CI (.github/workflows/validate-task.yml) and the VSCode tasks. [tool.ruff] target-version = "py313" From 134cb88f572a58dd860afa3e39c882249a0ae994 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Fri, 17 Jul 2026 09:18:16 -0700 Subject: [PATCH 09/12] Restore the Test Clone on Failure and Fix Two Nits Address Copilot review feedback: - RegressionTest.sh: roll the test clone back to its pristine @backup via an EXIT trap so an aborted process or plugin run under set -e never leaves the shared dataset mutated; the trap is disarmed after the clean restore. Also complete the mode-validation usage string with the corpus and plugin args. - locate_issue.py: write_region now sets the top-level schema field so a generated rules file matches the shipped reduction-rules.example.json. Co-Authored-By: Claude Opus 4.8 --- RegressionTests/RegressionTest.sh | 6 +++++- RegressionTests/locate_issue.py | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/RegressionTests/RegressionTest.sh b/RegressionTests/RegressionTest.sh index b339cd38..a4eb9d13 100755 --- a/RegressionTests/RegressionTest.sh +++ b/RegressionTests/RegressionTest.sh @@ -54,7 +54,7 @@ Mode="${1:-quick}" case "$Mode" in quick | full) ;; *) - echo "Usage: $0 [quick|full] [tag]" >&2 + echo "Usage: $0 [quick|full] [tag] [full|reduced] [plugin...]" >&2 exit 1 ;; esac @@ -360,11 +360,15 @@ RunRegressionTests() { # Provision a pristine clone of the corpus, process it, then run each built plugin on the # processed results to confirm it loads and runs; per-plugin behaviour is verified in isolation ProvisionDataset + # Roll the clone back to its pristine @backup even if processing or a plugin aborts under set -e, + # so a failed run never leaves the shared test dataset mutated; disarmed after the clean restore. + trap 'RestoreDataset' EXIT RunProcess "$Tag" "$Version" for Dll in "${BuiltPlugins[@]}"; do RunPlugin "$Tag" "$Version" "$Dll" done RestoreDataset + trap - EXIT } echo "Starting tests in $Mode mode" diff --git a/RegressionTests/locate_issue.py b/RegressionTests/locate_issue.py index b07e7015..3bbe19e6 100644 --- a/RegressionTests/locate_issue.py +++ b/RegressionTests/locate_issue.py @@ -142,6 +142,7 @@ def full_decode_locate(src: Path, needles: list[str]) -> float | None: def write_region(rules_path: Path, name: str, start: int, end: int, note: str) -> None: """Merge a region window for name into the external rules file (create if absent).""" data = json.loads(rules_path.read_text()) if rules_path.exists() else {} + data.setdefault("schema", 1) # keep the file self-describing, matching the shipped example data.setdefault("regions", {})[name] = {"start": start, "end": end, "note": note} rules_path.write_text(json.dumps(data, indent=2, ensure_ascii=False)) From 2ff2c98b726f1bbd87930e214e67a6d2169c9a25 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Fri, 17 Jul 2026 09:18:26 -0700 Subject: [PATCH 10/12] Pin Lint Tool Versions in CI Only, Latest Locally Keep the ruff and mypy version pins in the CI workflow (where they can be bumped) and run the latest tools in the VSCode tasks and the local-run examples in the README and pyproject comment. Pinning outside CI would drift silently with no bump path, which is worse than a small, visible local-versus-CI gap; a local linter that never falls behind is the safer default. Co-Authored-By: Claude Opus 4.8 --- .vscode/tasks.json | 6 +++--- RegressionTests/README.md | 12 ++++++------ RegressionTests/pyproject.toml | 10 ++++++---- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 0c09f1cc..1c39f20a 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -328,7 +328,7 @@ "type": "shell", "command": "uvx", "args": [ - "ruff@0.15.22", + "ruff", "check", "." ], @@ -346,7 +346,7 @@ "type": "shell", "command": "uvx", "args": [ - "ruff@0.15.22", + "ruff", "format", "--check", "." @@ -365,7 +365,7 @@ "type": "shell", "command": "uvx", "args": [ - "mypy@2.3.0", + "mypy", "." ], "options": { diff --git a/RegressionTests/README.md b/RegressionTests/README.md index dffca9e2..9889549f 100644 --- a/RegressionTests/README.md +++ b/RegressionTests/README.md @@ -173,11 +173,11 @@ are linted with ruff and type-checked with mypy; the configuration is in [`pyproject.toml`](pyproject.toml). Run them via `uvx`, which needs no install: ```shell -uvx ruff@0.15.22 check . -uvx ruff@0.15.22 format --check . -uvx mypy@2.3.0 . +uvx ruff check . +uvx ruff format --check . +uvx mypy . ``` -The versions are pinned so local runs match CI exactly; the same pinned commands run in CI and are -available as VSCode tasks. Python source is CRLF, matching the repository's default line-ending -convention. +These run the latest tools and are available as VSCode tasks; CI pins exact versions (bumpable +there), so local results may differ slightly - by design, so local tooling never silently falls +behind. Python source is CRLF, matching the repository's default line-ending convention. diff --git a/RegressionTests/pyproject.toml b/RegressionTests/pyproject.toml index e5d3b4dc..257e7cd9 100644 --- a/RegressionTests/pyproject.toml +++ b/RegressionTests/pyproject.toml @@ -4,11 +4,13 @@ # project/build metadata - only the ruff + mypy config, mirroring the ptr727 Financial-Modeling # conventions. Run the tools with uv (no install needed): # -# uvx ruff@0.15.22 check . -# uvx ruff@0.15.22 format --check . -# uvx mypy@2.3.0 . +# uvx ruff check . +# uvx ruff format --check . +# uvx mypy . # -# The versions are pinned to match CI (.github/workflows/validate-task.yml) and the VSCode tasks. +# These run the latest tools, matching the VSCode tasks; CI (.github/workflows/validate-task.yml) +# pins exact versions (bumpable there), so local results may differ slightly - by design, so local +# tooling never silently falls behind. [tool.ruff] target-version = "py313" From 46a8904583fda0154f14e502a66e180896dbd7c3 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Fri, 17 Jul 2026 09:23:22 -0700 Subject: [PATCH 11/12] Correct the CI Python-Lint Comment on Version Pinning The comment claimed the CI pins match the editor tasks, but the VSCode tasks now run the latest tools; only CI pins exact versions. Reword to state the intentional local-versus-CI gap. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/validate-task.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/validate-task.yml b/.github/workflows/validate-task.yml index 396bc0f5..d4096628 100644 --- a/.github/workflows/validate-task.yml +++ b/.github/workflows/validate-task.yml @@ -84,9 +84,10 @@ jobs: HISTORY.md incremental_files_only: false - # Lint the RegressionTests Python tooling with ruff + mypy, pinned to the same versions the - # editor tasks run, via uvx (no project install; the tooling is stdlib-only). Config lives in - # RegressionTests/pyproject.toml, so run from that directory (mypy resolves config from CWD). + # Lint the RegressionTests Python tooling with ruff + mypy via uvx (no project install; the + # tooling is stdlib-only). Versions are pinned here in CI (bumpable) for a reproducible gate; + # the VSCode tasks run the latest tools, so local may differ slightly by design. Config lives + # in RegressionTests/pyproject.toml, so run from that directory (mypy resolves config from CWD). - name: Setup uv step uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 From 28c53ef5b89c5aadb3b618e103a16c7cdc97681b Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Fri, 17 Jul 2026 09:31:36 -0700 Subject: [PATCH 12/12] Surface Container Output When a Clip Run Fails Address Copilot review feedback: process_clip captured the container stdout and stderr to DEVNULL, so a failed run (a crash or a container that never starts) gave no diagnostic. Capture the combined output and print its tail with the exit code when no results file is produced, or the parse error when the results file is unreadable, so a failed reduction rung is debuggable. Output is discarded on success. Co-Authored-By: Claude Opus 4.8 --- RegressionTests/reduce_corpus.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/RegressionTests/reduce_corpus.py b/RegressionTests/reduce_corpus.py index a1ef40db..3b8add7f 100644 --- a/RegressionTests/reduce_corpus.py +++ b/RegressionTests/reduce_corpus.py @@ -140,7 +140,9 @@ def process_clip(workdir: Path, settings_dir: Path) -> tuple[dict, dict[str, set media, out = workdir / "media", workdir / "out" out.mkdir(parents=True, exist_ok=True) uid, gid = os.getuid(), os.getgid() - subprocess.run( + # Capture the container output so a failed run (crash or container that never starts) can be + # diagnosed; on success it is discarded, on failure its tail is printed with the error below. + proc = subprocess.run( [ "docker", "run", @@ -165,17 +167,26 @@ def process_clip(workdir: Path, settings_dir: Path) -> tuple[dict, dict[str, set "--parallel", "--testsnippets", ], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + errors="replace", ) logmap = parse_log(out / "clip_process.log") results_file = out / "clip_results.json" if not results_file.exists(): + tail = "\n".join((proc.stdout or "").splitlines()[-15:]) + print( + f" run did not complete (exit {proc.returncode}); no results file:\n{tail}", + file=sys.stderr, + ) return logmap, None # run did not complete: not an empty result, a failure try: res = json.loads(results_file.read_text()) results = res["Results"]["Results"] - except (json.JSONDecodeError, KeyError, OSError): + except (json.JSONDecodeError, KeyError, OSError) as e: + print(f" results file present but unreadable: {e}", file=sys.stderr) return logmap, None states: dict[str, set[str]] = {} for r in results: