From 38a2c0bf177ad2a94b57357cee579bd734454b7a Mon Sep 17 00:00:00 2001 From: Simon Cropp Date: Sat, 11 Jul 2026 13:56:08 +1000 Subject: [PATCH 01/33] Add span based scrubber engine and Scrubber API Replaces the StringBuilder based scrubbing pipeline with a span based engine that materializes the document once, tracks it as chunks, and quarantines replacements so other scrubbers never reprocess them. New public API: Scrubber.Replace / Window / Match / RemoveLinesContaining / RemoveLines / ReplaceLines / RemoveEmptyLines, registered via AddScrubber(Scrubber) at global, instance, and extension mapped levels. - Ordering is engine determined: line drops, line transforms, then inline scrubbers (unknown max length first, then longest max first); directory replacements stay pinned last so scrubbers always see raw paths - Known min lengths skip short values; unchanged input is returned zero copy (the property value fast path now actually fires: the TempFile/TempDirectory module init scrubbers had made the old GlobalScrubbers.Count check permanently false) - All built-in scrub methods reroute to the engine; ScrubberLocation overloads are obsolete and ignored. Legacy AddScrubber(Action) is unchanged and runs after the engine, only when registered - Deletes ~900 lines of chunk carryover code (GuidScrubber, DateScrubber scan loops, LinesScrubber, UserMachineScrubber PerformReplacements, DirectoryReplacements_StringBuilder) - Date format validation probes 2000-01-01 instead of DateTime.MaxValue, which is out of range for the UmAlQura calendar Perf (31KB doc, Ryzen 5900X): guid scan 6.6x faster with 51x less allocation; ISO DateTime scan 6.8x / 100x; DateTimeOffset 11x; guids+dates+line drops composed in one pass 5.4x / 3.5x. New ApplyScrubbersTests project covers engine semantics; all suites pass with zero snapshot churn. --- docs/guids.md | 6 +- docs/mdsource/scrubbers.source.md | 25 +- docs/scrubbers.md | 25 +- .../ApplyScrubbersTests.csproj | 15 + src/ApplyScrubbersTests/EngineRunner.cs | 35 ++ src/ApplyScrubbersTests/EngineTests.cs | 155 ++++++ src/ApplyScrubbersTests/FastPathTests.cs | 107 ++++ src/ApplyScrubbersTests/GlobalUsings.cs | 2 + src/ApplyScrubbersTests/LegacyInteropTests.cs | 135 +++++ src/ApplyScrubbersTests/LinePhaseTests.cs | 139 +++++ src/ApplyScrubbersTests/MatchTests.cs | 143 +++++ src/ApplyScrubbersTests/OrderingTests.cs | 81 +++ src/ApplyScrubbersTests/WindowTests.cs | 88 +++ src/Benchmarks/FilterLinesBenchmarks.cs | 73 ++- src/Benchmarks/GlobalUsings.cs | 1 + .../InlineDateScrubberBenchmarks.cs | 91 +++- .../InlineGuidScrubberBenchmarks.cs | 127 ++++- src/Benchmarks/LegacyScrubbers.cs | 470 ++++++++++++++++ src/Benchmarks/LineScrubberBenchmarks.cs | 68 ++- .../UserMachineScrubberTests.cs | 9 +- src/Verify.Tests/DateScrubberTests.cs | 44 +- src/Verify.Tests/GuidScrubberTests.cs | 41 +- src/Verify.Tests/LinesScrubberTests.cs | 362 ++++--------- .../UserMachineScrubberChunkTests.cs | 38 -- src/Verify.slnx | 1 + src/Verify/Extensions.cs | 57 -- .../CustomContractResolver_Dictionary.cs | 2 +- .../Serialization/Scrubbers/ApplyScrubbers.cs | 101 +++- .../Serialization/Scrubbers/DateMatchers.cs | 204 +++++++ .../Serialization/Scrubbers/DateScrubber.cs | 274 ---------- .../Scrubbers/DirectoryReplacements.cs | 76 +++ .../Scrubbers/DirectoryReplacements_Span.cs | 105 ++++ .../DirectoryReplacements_StringBuilder.cs | 354 ------------ .../Scrubbers/EngineScrubberSet.cs | 212 ++++++++ .../Serialization/Scrubbers/GuidMatcher.cs | 30 + .../Serialization/Scrubbers/GuidScrubber.cs | 163 ------ .../Serialization/Scrubbers/LineResult.cs | 39 ++ .../Serialization/Scrubbers/LinesScrubber.cs | 40 -- .../Serialization/Scrubbers/ScrubEngine.cs | 512 ++++++++++++++++++ .../Scrubbers/ScrubEngine_Lines.cs | 231 ++++++++ .../Serialization/Scrubbers/Scrubber.cs | 283 ++++++++++ .../Serialization/Scrubbers/ScrubberKind.cs | 12 + .../Scrubbers/Scrubber_Delegates.cs | 34 ++ .../Scrubbers/UserMachineScrubber.cs | 12 +- ...UserMachineScrubber_PerformReplacements.cs | 144 ----- ...Settings_ExtensionMappedGlobalScrubbers.cs | 130 +++-- .../VerifierSettings_GlobalScrubbers.cs | 200 ++++--- ...ttings_ExtensionMappedInstanceScrubbers.cs | 99 +++- .../VerifySettings_InstanceScrubbers.cs | 159 ++++-- src/Verify/Serialization/VerifierSettings.cs | 4 + src/Verify/SettingsTask_Scrubbing.cs | 251 ++++++--- src/Verify/TempDirectory.cs | 51 +- src/Verify/TempFile.cs | 51 +- src/Verify/Verifier/InnerVerifier_Xml.cs | 11 +- src/Verify/VerifySettings.cs | 12 + 55 files changed, 4347 insertions(+), 1787 deletions(-) create mode 100644 src/ApplyScrubbersTests/ApplyScrubbersTests.csproj create mode 100644 src/ApplyScrubbersTests/EngineRunner.cs create mode 100644 src/ApplyScrubbersTests/EngineTests.cs create mode 100644 src/ApplyScrubbersTests/FastPathTests.cs create mode 100644 src/ApplyScrubbersTests/GlobalUsings.cs create mode 100644 src/ApplyScrubbersTests/LegacyInteropTests.cs create mode 100644 src/ApplyScrubbersTests/LinePhaseTests.cs create mode 100644 src/ApplyScrubbersTests/MatchTests.cs create mode 100644 src/ApplyScrubbersTests/OrderingTests.cs create mode 100644 src/ApplyScrubbersTests/WindowTests.cs create mode 100644 src/Benchmarks/LegacyScrubbers.cs delete mode 100644 src/Verify.Tests/UserMachineScrubberChunkTests.cs create mode 100644 src/Verify/Serialization/Scrubbers/DateMatchers.cs delete mode 100644 src/Verify/Serialization/Scrubbers/DateScrubber.cs create mode 100644 src/Verify/Serialization/Scrubbers/DirectoryReplacements_Span.cs delete mode 100644 src/Verify/Serialization/Scrubbers/DirectoryReplacements_StringBuilder.cs create mode 100644 src/Verify/Serialization/Scrubbers/EngineScrubberSet.cs create mode 100644 src/Verify/Serialization/Scrubbers/GuidMatcher.cs delete mode 100644 src/Verify/Serialization/Scrubbers/GuidScrubber.cs create mode 100644 src/Verify/Serialization/Scrubbers/LineResult.cs delete mode 100644 src/Verify/Serialization/Scrubbers/LinesScrubber.cs create mode 100644 src/Verify/Serialization/Scrubbers/ScrubEngine.cs create mode 100644 src/Verify/Serialization/Scrubbers/ScrubEngine_Lines.cs create mode 100644 src/Verify/Serialization/Scrubbers/Scrubber.cs create mode 100644 src/Verify/Serialization/Scrubbers/ScrubberKind.cs create mode 100644 src/Verify/Serialization/Scrubbers/Scrubber_Delegates.cs delete mode 100644 src/Verify/Serialization/Scrubbers/UserMachineScrubber_PerformReplacements.cs diff --git a/docs/guids.md b/docs/guids.md index a99d32809c..23312e2970 100644 --- a/docs/guids.md +++ b/docs/guids.md @@ -161,7 +161,7 @@ public Task NamedGuidInstance() settings); } ``` -snippet source | anchor +snippet source | anchor @@ -182,7 +182,7 @@ public Task NamedGuidFluent() .AddNamedGuid(guid, "instanceNamed"); } ``` -snippet source | anchor +snippet source | anchor @@ -217,7 +217,7 @@ public Task InferredNamedGuidFluent() .AddNamedGuid(namedGuid); } ``` -snippet source | anchor +snippet source | anchor Result: diff --git a/docs/mdsource/scrubbers.source.md b/docs/mdsource/scrubbers.source.md index d14f95c394..2134bcd2d0 100644 --- a/docs/mdsource/scrubbers.source.md +++ b/docs/mdsource/scrubbers.source.md @@ -4,9 +4,30 @@ Scrubbers run on the final string before doing the verification action. Multiple scrubbers [can be defined at multiple levels](#Scrubber-levels). -By default scrubber are executed in reverse order. So the most recent added method scrubber through to earliest added global scrubber. All scrubber APIs support a `ScrubberLocation location`. To execute a scrubber last use `ScrubberLocation.Last`. +Scrubbing is performed by two mechanisms: -Scrubbers can be added multiple times to have them execute multiple times. This can be helpful when compounding multiple scrubbers together. + * **The scrub engine**: a span based engine that executes `Scrubber` definitions. All built-in scrubbing (`ScrubLinesContaining`, `ScrubInlineGuids`, `ScrubMachineName`, etc) runs on the engine. + * **Legacy scrubbers**: `AddScrubber(Action)` overloads. These run after the engine, and only when at least one is registered. + + +## The scrub engine + +A `Scrubber` is created via static factory methods and registered via `AddScrubber` at any [level](#Scrubber-levels): + + * `Scrubber.Replace(find, replacement)`: replace every occurrence of a string. Supports a `StringComparison` and an optional word boundary requirement. A multi-pair overload replaces the longest matching find at any position. + * `Scrubber.Window(minLength, maxLength, matcher)`: slide a window over the text; the matcher returns a replacement or null. Used by the inline guid and date scrubbers. + * `Scrubber.Match(matcher, minLength, maxLength)`: custom search logic. The matcher locates the next match within a segment. + * `Scrubber.RemoveLinesContaining(...)`, `Scrubber.RemoveLines(...)`, `Scrubber.ReplaceLines(...)`, `Scrubber.RemoveEmptyLines()`: line scoped scrubbers. + +Engine semantics: + + * **Quarantine**: text produced by a replacement is never re-examined by other engine scrubbers. Legacy scrubbers run afterwards and can still modify it. + * **Ordering** is engine determined, not registration determined: line removals run first, then line transforms (registration order), then inline scrubbers (unknown max length first, then longest max length first, ties broken by level then registration order). Directory replacements always run last, so scrubbers always see raw paths. + * **Length skip**: text shorter than a scrubber's minimum match length is never scanned by that scrubber. + * **Single line rule**: a match may never contain a line break. + * Text is newline normalized (`\r\n` and `\r` become `\n`) before scrubbers run. + +`ScrubberLocation` on the built-in scrub methods is obsolete and ignored; it still applies to legacy `AddScrubber` overloads (default `First` executes in reverse registration order, `Last` in registration order). ## Available Scrubbers diff --git a/docs/scrubbers.md b/docs/scrubbers.md index 9ea9c2573a..b15f944ad6 100644 --- a/docs/scrubbers.md +++ b/docs/scrubbers.md @@ -11,9 +11,30 @@ Scrubbers run on the final string before doing the verification action. Multiple scrubbers [can be defined at multiple levels](#Scrubber-levels). -By default scrubber are executed in reverse order. So the most recent added method scrubber through to earliest added global scrubber. All scrubber APIs support a `ScrubberLocation location`. To execute a scrubber last use `ScrubberLocation.Last`. +Scrubbing is performed by two mechanisms: -Scrubbers can be added multiple times to have them execute multiple times. This can be helpful when compounding multiple scrubbers together. + * **The scrub engine**: a span based engine that executes `Scrubber` definitions. All built-in scrubbing (`ScrubLinesContaining`, `ScrubInlineGuids`, `ScrubMachineName`, etc) runs on the engine. + * **Legacy scrubbers**: `AddScrubber(Action)` overloads. These run after the engine, and only when at least one is registered. + + +## The scrub engine + +A `Scrubber` is created via static factory methods and registered via `AddScrubber` at any [level](#Scrubber-levels): + + * `Scrubber.Replace(find, replacement)`: replace every occurrence of a string. Supports a `StringComparison` and an optional word boundary requirement. A multi-pair overload replaces the longest matching find at any position. + * `Scrubber.Window(minLength, maxLength, matcher)`: slide a window over the text; the matcher returns a replacement or null. Used by the inline guid and date scrubbers. + * `Scrubber.Match(matcher, minLength, maxLength)`: custom search logic. The matcher locates the next match within a segment. + * `Scrubber.RemoveLinesContaining(...)`, `Scrubber.RemoveLines(...)`, `Scrubber.ReplaceLines(...)`, `Scrubber.RemoveEmptyLines()`: line scoped scrubbers. + +Engine semantics: + + * **Quarantine**: text produced by a replacement is never re-examined by other engine scrubbers. Legacy scrubbers run afterwards and can still modify it. + * **Ordering** is engine determined, not registration determined: line removals run first, then line transforms (registration order), then inline scrubbers (unknown max length first, then longest max length first, ties broken by level then registration order). Directory replacements always run last, so scrubbers always see raw paths. + * **Length skip**: text shorter than a scrubber's minimum match length is never scanned by that scrubber. + * **Single line rule**: a match may never contain a line break. + * Text is newline normalized (`\r\n` and `\r` become `\n`) before scrubbers run. + +`ScrubberLocation` on the built-in scrub methods is obsolete and ignored; it still applies to legacy `AddScrubber` overloads (default `First` executes in reverse registration order, `Last` in registration order). ## Available Scrubbers diff --git a/src/ApplyScrubbersTests/ApplyScrubbersTests.csproj b/src/ApplyScrubbersTests/ApplyScrubbersTests.csproj new file mode 100644 index 0000000000..23383b3f2e --- /dev/null +++ b/src/ApplyScrubbersTests/ApplyScrubbersTests.csproj @@ -0,0 +1,15 @@ + + + net11.0 + false + Exe + + + + + + + + + + diff --git a/src/ApplyScrubbersTests/EngineRunner.cs b/src/ApplyScrubbersTests/EngineRunner.cs new file mode 100644 index 0000000000..738b2b4c3a --- /dev/null +++ b/src/ApplyScrubbersTests/EngineRunner.cs @@ -0,0 +1,35 @@ +static class EngineRunner +{ + public static Dictionary EmptyContext { get; } = []; + + public static string Run(string input, params Scrubber[] scrubbers) + { + using var counter = Counter.Start(); + return Run(input, counter, scrubbers); + } + + public static string Run(string input, Counter counter, params Scrubber[] scrubbers) => + ScrubEngine.Run( + input, + EngineScrubberSet.ForScrubbers([.. scrubbers]), + counter, + EmptyContext, + applyDirectoryReplacements: false); + + public static string RunWithDirectoryReplacements(string input, params Scrubber[] scrubbers) + { + using var counter = Counter.Start(); + return ScrubEngine.Run( + input, + EngineScrubberSet.ForScrubbers([.. scrubbers]), + counter, + EmptyContext, + applyDirectoryReplacements: true); + } + + // Deterministic path replacements, matching the DisableScrubbersTests pattern. + // Called from static ctors of test classes that exercise path replacement, so it + // runs after the adapter has assigned the real target assembly paths. + public static void UseFakeDirectories() => + DirectoryReplacements.UseAssembly("C:/Code/TheSolution", "C:/Code/TheSolution/TheProject"); +} diff --git a/src/ApplyScrubbersTests/EngineTests.cs b/src/ApplyScrubbersTests/EngineTests.cs new file mode 100644 index 0000000000..6f9f602962 --- /dev/null +++ b/src/ApplyScrubbersTests/EngineTests.cs @@ -0,0 +1,155 @@ +public class EngineTests +{ + [Fact] + public void Quarantine_ReplacementNotSeenByOtherScrubbers() + { + // Equal max lengths, so registration order applies: first replaces abc + // with xyz, and the quarantined xyz must not be re-matched by the second + var result = EngineRunner.Run( + "abc", + Scrubber.Replace("abc", "xyz"), + Scrubber.Replace("xyz", "!!!")); + Assert.Equal("xyz", result); + } + + [Fact] + public void Quarantine_ReplacementNotRescannedBySameScrubber() + { + // A replacement containing its own find must not recurse + var result = EngineRunner.Run("aa", Scrubber.Replace("aa", "aaaa")); + Assert.Equal("aaaa", result); + } + + [Fact] + public void ZeroCopy_NoMatches() + { + var input = "no matches here"; + var result = EngineRunner.Run(input, Scrubber.Replace("missing", "x")); + Assert.Same(input, result); + } + + [Fact] + public void Newlines_CrLfCollapsed() + { + var result = EngineRunner.Run("a\r\nb\rc", Scrubber.Replace("missing", "x")); + Assert.Equal("a\nb\nc", result); + } + + [Fact] + public void Newlines_ReplacementNormalized() + { + var result = EngineRunner.Run( + "token", + Scrubber.Match((CharSpan segment, Counter _, IReadOnlyDictionary _, out int index, out int length, out string? replacement) => + { + index = segment.IndexOf("token".AsSpan()); + if (index < 0) + { + length = 0; + replacement = null; + return false; + } + + length = 5; + replacement = "x\r\ny"; + return true; + })); + Assert.Equal("x\ny", result); + } + + [Fact] + public void EmptyInput() + { + var input = string.Empty; + var result = EngineRunner.Run(input, Scrubber.Replace("a", "b")); + Assert.Same(input, result); + } + + [Fact] + public void Split_PrefixAndSuffixPreserved() + { + var result = EngineRunner.Run("xxabcxx", Scrubber.Replace("abc", "Y")); + Assert.Equal("xxYxx", result); + } + + [Fact] + public void EmptyReplacement_RemovesText() + { + var result = EngineRunner.Run("a-remove-b", Scrubber.Replace("-remove-", "")); + Assert.Equal("ab", result); + } + + [Fact] + public void MultipleMatches_SameScrubber() + { + var result = EngineRunner.Run("abc abc abc", Scrubber.Replace("abc", "Y")); + Assert.Equal("Y Y Y", result); + } + + [Fact] + public void WordBoundary_RejectsAdjacentLetterOrDigit() + { + var result = EngineRunner.Run( + "name names 1name name", + Scrubber.Replace("name", "X", requireWordBoundary: true)); + Assert.Equal("X names 1name X", result); + } + + [Fact] + public void WordBoundary_ReplacementChunkIsNeighbor() + { + // First scrubber replaces ab with 12. The second requires a word boundary; + // the preceding chunk now ends with '2' (a digit), so cd must not match. + var result = EngineRunner.Run( + "abcd", + Scrubber.Replace("ab", "12"), + Scrubber.Replace("cd", "YZ", requireWordBoundary: true)); + Assert.Equal("12cd", result); + } + + [Fact] + public void MinLength_SkipsShortValues() + { + var input = "ab"; + var result = EngineRunner.Run(input, Scrubber.Replace("abc", "x")); + Assert.Same(input, result); + } + + [Fact] + public void MultiPair_LongestWinsAtSamePosition() + { + var result = EngineRunner.Run( + "abc", + Scrubber.Replace(StringComparison.Ordinal, false, ("ab", "SHORT"), ("abc", "LONG"))); + Assert.Equal("LONG", result); + } + + [Fact] + public void MultiPair_EarlierPositionWins() + { + var result = EngineRunner.Run( + "abc", + Scrubber.Replace(StringComparison.Ordinal, false, ("bc", "B"), ("c", "C"))); + Assert.Equal("aB", result); + } + + [Fact] + public void Comparison_OrdinalIgnoreCase() + { + var result = EngineRunner.Run( + "ABC abc", + Scrubber.Replace("abc", "x", StringComparison.OrdinalIgnoreCase)); + Assert.Equal("x x", result); + } + + [Fact] + public void Validation_FindWithNewlineThrows() => + Assert.Throws(() => Scrubber.Replace("a\nb", "x")); + + [Fact] + public void Validation_WindowBoundsThrow() + { + Assert.Throws(() => Scrubber.Window(0, 5, (CharSpan _, Counter _, IReadOnlyDictionary _) => null)); + Assert.Throws(() => Scrubber.Window(5, 4, (CharSpan _, Counter _, IReadOnlyDictionary _) => null)); + } +} diff --git a/src/ApplyScrubbersTests/FastPathTests.cs b/src/ApplyScrubbersTests/FastPathTests.cs new file mode 100644 index 0000000000..31007c0078 --- /dev/null +++ b/src/ApplyScrubbersTests/FastPathTests.cs @@ -0,0 +1,107 @@ +public class FastPathTests +{ + static FastPathTests() => + EngineRunner.UseFakeDirectories(); + + static string Apply(string value, Action? configure = null) + { + var settings = new VerifySettings(); + configure?.Invoke(settings); + using var counter = Counter.Start(); + return ApplyScrubbers.ApplyForPropertyValue(value, settings, counter); + } + + [Fact] + public void ShortValue_NoScrubbers_SameInstance() + { + var value = "short"; + Assert.Same(value, Apply(value)); + } + + [Fact] + public void CarriageReturn_ForcesNormalization() + { + var value = "a\r\nb"; + var result = Apply(value); + Assert.Equal("a\nb", result); + } + + [Fact] + public void ValueShorterThanScrubberMin_SameInstance() + { + var value = "ab"; + var result = Apply(value, settings => settings.AddScrubber(Scrubber.Replace("abcdef", "x"))); + Assert.Same(value, result); + } + + [Fact] + public void MatchingValue_Scrubbed() + { + var result = Apply("abcdef!", settings => settings.AddScrubber(Scrubber.Replace("abcdef", "x"))); + Assert.Equal("x!", result); + } + + [Fact] + public void UnknownMinScrubber_DisablesFastPath() + { + var invoked = false; + Apply( + "a", + settings => settings.AddScrubber(Scrubber.Match( + (CharSpan _, Counter _, IReadOnlyDictionary _, out int index, out int length, out string? replacement) => + { + invoked = true; + index = 0; + length = 0; + replacement = null; + return false; + }))); + Assert.True(invoked); + } + + [Fact] + public void PathValue_Replaced() + { + var result = Apply("C:/Code/TheSolution/TheProject/file.txt"); + Assert.Equal("{ProjectDirectory}file.txt", result); + } + + [Fact] + public void LegacyScrubber_ForcesFullPath() + { + var result = Apply( + "ab", + settings => settings.AddScrubber(builder => builder.Append('!'))); + Assert.Equal("ab!", result); + } + + [Fact] + public void ExtensionMappedScrubbers_ExcludedFromPropertyPath() + { + var value = "abc"; + var result = Apply(value, settings => settings.AddScrubber("txt", Scrubber.Replace("abc", "xyz"))); + Assert.Same(value, result); + } + + [Fact] + public void ScrubbersDisabled_OnlyNormalizes() + { + var result = Apply( + "abc\r\n", + settings => + { + settings.AddScrubber(Scrubber.Replace("abc", "xyz")); + settings.DisableScrubbers(); + }); + Assert.Equal("abc\n", result); + } + + [Fact] + public void NoMatch_EngineStillZeroCopy() + { + // Long enough to enter the engine, but nothing matches: same instance back + var value = new string('x', 200); + var result = Apply(value, settings => settings.AddScrubber(Scrubber.Replace("missing", "y"))); + Assert.Same(value, result); + } +} diff --git a/src/ApplyScrubbersTests/GlobalUsings.cs b/src/ApplyScrubbersTests/GlobalUsings.cs new file mode 100644 index 0000000000..16b8752896 --- /dev/null +++ b/src/ApplyScrubbersTests/GlobalUsings.cs @@ -0,0 +1,2 @@ +global using System.Text; +global using CharSpan = System.ReadOnlySpan; diff --git a/src/ApplyScrubbersTests/LegacyInteropTests.cs b/src/ApplyScrubbersTests/LegacyInteropTests.cs new file mode 100644 index 0000000000..faa1dc010c --- /dev/null +++ b/src/ApplyScrubbersTests/LegacyInteropTests.cs @@ -0,0 +1,135 @@ +public class LegacyInteropTests +{ + static LegacyInteropTests() => + EngineRunner.UseFakeDirectories(); + + static string RunForExtension(string input, Action configure) + { + var settings = new VerifySettings(); + configure(settings); + using var counter = Counter.Start(); + var builder = new StringBuilder(input); + ApplyScrubbers.ApplyForExtension("txt", builder, settings, counter); + return builder.ToString(); + } + + [Fact] + public void LegacyRunsAfterEngine() + { + // The span scrubber quarantines its replacement from other span scrubbers, + // but the legacy pass still sees and may modify it + var result = RunForExtension( + "a", + settings => + { + settings.AddScrubber(Scrubber.Replace("a", "1")); + settings.AddScrubber(builder => builder.Replace("1", "2")); + }); + Assert.Equal("2", result); + } + + [Fact] + public void LegacyLocations_StillHonored() + { + var result = RunForExtension( + "value", + settings => + { + settings.AddScrubber(builder => builder.Append(" one"), ScrubberLocation.Last); + settings.AddScrubber(builder => builder.Append(" two"), ScrubberLocation.Last); + }); + Assert.Equal("value one two", result); + } + + [Fact] + public void FixNewlines_AfterLegacy() + { + var result = RunForExtension( + "value", + settings => settings.AddScrubber(builder => builder.Append("\r\nline"))); + Assert.Equal("value\nline", result); + } + + [Fact] + public void PathReplacements_AfterLegacy() + { + // A legacy scrubber writes a raw path; the trailing path replacement pass + // still converts it + var result = RunForExtension( + "value ", + settings => settings.AddScrubber(builder => builder.Append("C:/Code/TheSolution/TheProject/file.txt"))); + Assert.Equal("value {ProjectDirectory}file.txt", result); + } + + [Fact] + public void LegacyUserScrubber_BeatsPathReplacements() + { + // MoreSpecificScrubberShouldOverride semantics: the legacy scrubber sees the + // raw path before the path replacement pass + var result = RunForExtension( + "C:/Code/TheSolution/TheProject/data", + settings => settings.AddScrubber(builder => builder.Replace("C:/Code/TheSolution/TheProject/data", "{custom}"))); + Assert.Equal("{custom}", result); + } + + [Fact] + public void NoLegacy_EngineOnlyPath() + { + var result = RunForExtension( + "C:/Code/TheSolution/TheProject/file.txt abc", + settings => settings.AddScrubber(Scrubber.Replace("abc", "xyz"))); + Assert.Equal("{ProjectDirectory}file.txt xyz", result); + } + + [Fact] + public void ExtensionMapped_SpanScrubberApplies() + { + var result = RunForExtension( + "abc", + settings => settings.AddScrubber("txt", Scrubber.Replace("abc", "xyz"))); + Assert.Equal("xyz", result); + } + + [Fact] + public void ExtensionMapped_OtherExtensionIgnored() + { + var result = RunForExtension( + "abc", + settings => settings.AddScrubber("json", Scrubber.Replace("abc", "xyz"))); + Assert.Equal("abc", result); + } + + [Fact] + public void DisableScrubbers_BypassesEverything() + { + var result = RunForExtension( + "abc\r\n", + settings => + { + settings.AddScrubber(Scrubber.Replace("abc", "xyz")); + settings.DisableScrubbers(); + }); + Assert.Equal("abc\n", result); + } + + [Fact] + public void SettingsClone_CopiesSpanScrubbers() + { + var settings = new VerifySettings(); + settings.AddScrubber(Scrubber.Replace("abc", "xyz")); + settings.AddScrubber("txt", Scrubber.Replace("def", "uvw")); + + var clone = new VerifySettings(settings); + using var counter = Counter.Start(); + var builder = new StringBuilder("abc def"); + ApplyScrubbers.ApplyForExtension("txt", builder, clone, counter); + Assert.Equal("xyz uvw", builder.ToString()); + + // Mutating the clone must not affect the original + clone.AddScrubber(Scrubber.Replace("ghi", "rst")); + using var secondCounter = Counter.Start(); + var second = new StringBuilder("ghi"); + ApplyScrubbers.ApplyForExtension("txt", second, settings, secondCounter); + Assert.Equal("ghi", second.ToString()); + } +} diff --git a/src/ApplyScrubbersTests/LinePhaseTests.cs b/src/ApplyScrubbersTests/LinePhaseTests.cs new file mode 100644 index 0000000000..df18513734 --- /dev/null +++ b/src/ApplyScrubbersTests/LinePhaseTests.cs @@ -0,0 +1,139 @@ +public class LinePhaseTests +{ + [Fact] + public void Transforms_ComposeInRegistrationOrder() + { + var result = EngineRunner.Run( + "a", + Scrubber.ReplaceLines(_ => _ == "a" ? "b" : _), + Scrubber.ReplaceLines(_ => _ == "b" ? "c" : _)); + Assert.Equal("c", result); + } + + [Fact] + public void Transform_LineResultVariants() + { + var result = EngineRunner.Run( + "keep\nremove\nreplace", + Scrubber.ReplaceLines(line => + { + if (line.SequenceEqual("remove".AsSpan())) + { + return LineResult.Remove; + } + + if (line.SequenceEqual("replace".AsSpan())) + { + return LineResult.Replace("replaced"); + } + + return LineResult.Keep; + })); + Assert.Equal("keep\nreplaced", result); + } + + [Fact] + public void Transform_NullDrops() + { + var result = EngineRunner.Run( + "keep\ndrop me", + Scrubber.ReplaceLines(_ => _.Contains("drop") ? null : _)); + Assert.Equal("keep", result); + } + + [Fact] + public void TransformedLine_RescannedByInlineScrubbers() + { + var result = EngineRunner.Run( + "line", + Scrubber.ReplaceLines(_ => _ == "line" ? "token here" : _), + Scrubber.Replace("token", "X")); + Assert.Equal("X here", result); + } + + [Fact] + public void Drops_SeeRawLines_NotTransformOutput() + { + // The transform would rewrite the marker, but drops run first on the raw line + var result = EngineRunner.Run( + "clean\nsecret data", + Scrubber.ReplaceLines(_ => _.Replace("secret", "public")), + Scrubber.RemoveLinesContaining(StringComparison.Ordinal, "secret")); + Assert.Equal("clean", result); + } + + [Fact] + public void NeedleDrop_LengthSkip() + { + // Lines shorter than the needle cannot match and are kept + var result = EngineRunner.Run( + "ab\nlongneedle here\ncd", + Scrubber.RemoveLinesContaining(StringComparison.Ordinal, "longneedle")); + Assert.Equal("ab\ncd", result); + } + + [Fact] + public void ComparisonBucketing() + { + var result = EngineRunner.Run( + "foo\nFOO\nbar\nBAR\nkeep", + Scrubber.RemoveLinesContaining(StringComparison.Ordinal, "foo"), + Scrubber.RemoveLinesContaining("BAR")); + Assert.Equal("FOO\nkeep", result); + } + + [Fact] + public void RemoveEmptyLines_DropsWhitespaceLines() + { + var result = EngineRunner.Run( + "a\n\n \t\nb", + Scrubber.RemoveEmptyLines()); + Assert.Equal("a\nb", result); + } + + [Fact] + public void RemoveEmptyLines_TrimsTrailingNewline() + { + // Matches legacy RemoveEmptyLines: the trailing newline is trimmed even when + // no line was dropped + var result = EngineRunner.Run( + "a\nb\n", + Scrubber.RemoveEmptyLines()); + Assert.Equal("a\nb", result); + } + + [Fact] + public void SpanPredicateOverload() + { + var result = EngineRunner.Run( + "keep\nremove", + Scrubber.RemoveLines((LineMatch) (line => line.StartsWith("remove".AsSpan())))); + Assert.Equal("keep", result); + } + + [Fact] + public void DropFirstMiddleLast() + { + Assert.Equal("b\nc", EngineRunner.Run("x\nb\nc", Scrubber.RemoveLinesContaining(StringComparison.Ordinal, "x"))); + Assert.Equal("a\nc", EngineRunner.Run("a\nx\nc", Scrubber.RemoveLinesContaining(StringComparison.Ordinal, "x"))); + Assert.Equal("a\nb", EngineRunner.Run("a\nb\nx", Scrubber.RemoveLinesContaining(StringComparison.Ordinal, "x"))); + Assert.Equal("", EngineRunner.Run("x\nx\nx", Scrubber.RemoveLinesContaining(StringComparison.Ordinal, "x"))); + } + + [Fact] + public void TrailingNewlinePreservedOnDrop() + { + Assert.Equal("a\n", EngineRunner.Run("a\nx\n", Scrubber.RemoveLinesContaining(StringComparison.Ordinal, "x"))); + Assert.Equal("a", EngineRunner.Run("a\nx", Scrubber.RemoveLinesContaining(StringComparison.Ordinal, "x"))); + } + + [Fact] + public void LineResultReplace_NormalizesNewlines() + { + var replaced = LineResult.Replace("a\r\nb"); + var result = EngineRunner.Run( + "line", + Scrubber.ReplaceLines(_ => replaced)); + Assert.Equal("a\nb", result); + } +} diff --git a/src/ApplyScrubbersTests/MatchTests.cs b/src/ApplyScrubbersTests/MatchTests.cs new file mode 100644 index 0000000000..39051ed659 --- /dev/null +++ b/src/ApplyScrubbersTests/MatchTests.cs @@ -0,0 +1,143 @@ +public class MatchTests +{ + static SegmentMatch DigitRunMatcher { get; } = + (CharSpan segment, Counter _, IReadOnlyDictionary _, out int index, out int length, out string? replacement) => + { + for (index = 0; index < segment.Length; index++) + { + if (!char.IsDigit(segment[index])) + { + continue; + } + + var end = index; + while (end < segment.Length && + char.IsDigit(segment[end])) + { + end++; + } + + length = end - index; + replacement = "#"; + return true; + } + + length = 0; + replacement = null; + return false; + }; + + [Fact] + public void ReinvokedPastEachMatch() + { + var result = EngineRunner.Run("a1b22c333", Scrubber.Match(DigitRunMatcher)); + Assert.Equal("a#b#c#", result); + } + + [Fact] + public void MinLength_SkipsShortSegments() + { + var invoked = false; + EngineRunner.Run( + "abc", + Scrubber.Match( + (CharSpan _, Counter _, IReadOnlyDictionary _, out int index, out int length, out string? replacement) => + { + invoked = true; + index = 0; + length = 0; + replacement = null; + return false; + }, + minLength: 5)); + Assert.False(invoked); + } + + [Fact] + public void InvalidRange_Throws() + { + var exception = Assert.ThrowsAny(() => + EngineRunner.Run( + "abc", + Scrubber.Match((CharSpan _, Counter _, IReadOnlyDictionary _, out int index, out int length, out string? replacement) => + { + index = 0; + length = 10; + replacement = "x"; + return true; + }))); + Assert.Contains("invalid range", exception.Message); + } + + [Fact] + public void NewlineSpanningMatch_Throws() + { + var exception = Assert.ThrowsAny(() => + EngineRunner.Run( + "ab\ncd", + Scrubber.Match((CharSpan _, Counter _, IReadOnlyDictionary _, out int index, out int length, out string? replacement) => + { + index = 0; + length = 5; + replacement = "x"; + return true; + }))); + Assert.Contains("line break", exception.Message); + } + + [Fact] + public void ContextAndCounter_FlowToMatcher() + { + using var counter = Counter.Start(); + Counter? seenCounter = null; + IReadOnlyDictionary? seenContext = null; + EngineRunner.Run( + "abc", + counter, + Scrubber.Match((CharSpan _, Counter matcherCounter, IReadOnlyDictionary matcherContext, out int index, out int length, out string? replacement) => + { + seenCounter = matcherCounter; + seenContext = matcherContext; + index = 0; + length = 0; + replacement = null; + return false; + })); + Assert.Same(counter, seenCounter); + Assert.Same(EngineRunner.EmptyContext, seenContext); + } + + [Fact] + public void UlidStyle_CounterNumbering() + { + // A fixed length token matcher using counter numbering, the Verify.Ulid shape + using var counter = Counter.Start(); + var result = EngineRunner.Run( + "01ARZ3NDEKTSV4RRFFQ69G5FAV then 01BX5ZZKBKACTAV9WEVGEMMVRZ", + counter, + Scrubber.Window(26, 26, (window, windowCounter, _) => + { + foreach (var ch in window) + { + if (!char.IsLetterOrDigit(ch)) + { + return null; + } + } + + return $"Ulid_{windowCounter.Next(new Guid(HashUlid(window), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0))}"; + }, requireWordBoundary: true)); + Assert.Equal("Ulid_1 then Ulid_2", result); + } + + static int HashUlid(CharSpan window) + { + var hash = 17; + foreach (var ch in window) + { + hash = unchecked((hash * 31) + ch); + } + + return hash; + } +} diff --git a/src/ApplyScrubbersTests/OrderingTests.cs b/src/ApplyScrubbersTests/OrderingTests.cs new file mode 100644 index 0000000000..efcf8333d8 --- /dev/null +++ b/src/ApplyScrubbersTests/OrderingTests.cs @@ -0,0 +1,81 @@ +public class OrderingTests +{ + static OrderingTests() => + EngineRunner.UseFakeDirectories(); + + [Fact] + public void UnknownMaxRunsFirst() + { + // The Match scrubber has no max length, so it runs before the known length + // Replace even though the Replace was registered first + var result = EngineRunner.Run( + "abcdef", + Scrubber.Replace("abcdef", "R"), + Scrubber.Match((CharSpan segment, Counter _, IReadOnlyDictionary _, out int index, out int length, out string? replacement) => + { + index = segment.IndexOf("abc".AsSpan()); + if (index < 0) + { + length = 0; + replacement = null; + return false; + } + + length = 3; + replacement = "M"; + return true; + })); + Assert.Equal("Mdef", result); + } + + [Fact] + public void LongestMaxRunsFirst() + { + // Registered shortest first; the longer max must still win the overlap + var result = EngineRunner.Run( + "abc", + Scrubber.Replace("bc", "X"), + Scrubber.Replace("abc", "Y")); + Assert.Equal("Y", result); + } + + [Fact] + public void EqualMax_RegistrationOrderWins() + { + var result = EngineRunner.Run( + "ab", + Scrubber.Replace("ab", "1"), + Scrubber.Replace("ab", "2")); + Assert.Equal("1", result); + } + + [Fact] + public void PathReplacementsApply() + { + var result = EngineRunner.RunWithDirectoryReplacements("C:/Code/TheSolution/TheProject/file.txt"); + Assert.Equal("{ProjectDirectory}file.txt", result); + } + + [Fact] + public void UserScrubberBeatsPathReplacements() + { + // Path replacements are pinned last: a user scrubber matching the raw path + // wins, mirroring MoreSpecificScrubberShouldOverride + var result = EngineRunner.RunWithDirectoryReplacements( + "C:/Code/TheSolution/TheProject/file.txt", + Scrubber.Replace("C:/Code/TheSolution/TheProject/file.txt", "{custom}")); + Assert.Equal("{custom}", result); + } + + [Fact] + public void PathReplacements_RespectQuarantine() + { + // The user scrubber consumes the project segment, so the full project path + // can no longer match; the remaining raw prefix still matches the solution + // directory (greedily absorbing the trailing separator) + var result = EngineRunner.RunWithDirectoryReplacements( + "C:/Code/TheSolution/TheProject", + Scrubber.Replace("TheProject", "{p}")); + Assert.Equal("{SolutionDirectory}{p}", result); + } +} diff --git a/src/ApplyScrubbersTests/WindowTests.cs b/src/ApplyScrubbersTests/WindowTests.cs new file mode 100644 index 0000000000..9a180879a6 --- /dev/null +++ b/src/ApplyScrubbersTests/WindowTests.cs @@ -0,0 +1,88 @@ +public class WindowTests +{ + [Fact] + public void FixedLength() + { + var result = EngineRunner.Run( + "xx ABC xx", + Scrubber.Window(3, 3, (window, _, _) => window.SequenceEqual("ABC".AsSpan()) ? "match" : null)); + Assert.Equal("xx match xx", result); + } + + [Fact] + public void VariableLength_LongestTriedFirst() + { + var lengths = new List(); + EngineRunner.Run( + "abcdefgh", + Scrubber.Window(2, 4, (window, _, _) => + { + lengths.Add(window.Length); + return null; + })); + + // At the first position all lengths fit and are probed longest first + Assert.Equal([4, 3, 2], lengths.Take(3)); + } + + [Fact] + public void WindowsNeverContainNewlines() + { + var windows = new List(); + EngineRunner.Run( + "ab\ncd", + Scrubber.Window(2, 4, (window, _, _) => + { + windows.Add(window.ToString()); + return null; + })); + + Assert.NotEmpty(windows); + Assert.All(windows, _ => Assert.DoesNotContain('\n', _)); + } + + [Fact] + public void WordBoundary_DocumentEdgesAreValid() + { + var result = EngineRunner.Run( + "ABC", + Scrubber.Window(3, 3, (window, _, _) => window.SequenceEqual("ABC".AsSpan()) ? "match" : null, requireWordBoundary: true)); + Assert.Equal("match", result); + } + + [Fact] + public void WordBoundary_AdjacentLetterRejects() + { + var result = EngineRunner.Run( + "xABC ABCx ABC", + Scrubber.Window(3, 3, (window, _, _) => window.SequenceEqual("ABC".AsSpan()) ? "match" : null, requireWordBoundary: true)); + Assert.Equal("xABC ABCx match", result); + } + + [Fact] + public void Counter_FlowsToMatcher() + { + using var counter = Counter.Start(); + Counter? seen = null; + EngineRunner.Run( + "abc", + counter, + Scrubber.Window(3, 3, (_, matcherCounter, _) => + { + seen = matcherCounter; + return null; + })); + Assert.Same(counter, seen); + } + + [Fact] + public void GuidWindow_EndToEnd() + { + using var counter = Counter.Start(); + var result = EngineRunner.Run( + "id: 173535ae-995b-4cc6-a74e-8cd4be57039c done", + counter, + GuidMatcher.Instance); + Assert.Equal("id: Guid_1 done", result); + } +} diff --git a/src/Benchmarks/FilterLinesBenchmarks.cs b/src/Benchmarks/FilterLinesBenchmarks.cs index 7143720381..bdc20338ef 100644 --- a/src/Benchmarks/FilterLinesBenchmarks.cs +++ b/src/Benchmarks/FilterLinesBenchmarks.cs @@ -1,10 +1,17 @@ -[MemoryDiagnoser] +// Predicate based line removal (ScrubLines). Legacy_* rows run the pre-engine +// StringReader implementation; Engine_* rows run the span engine line phase with a +// string predicate (the engine materializes each line lazily for the predicate). +[MemoryDiagnoser] [SimpleJob(iterationCount: 10, warmupCount: 3)] public class FilterLinesBenchmarks { - StringBuilder smallInput = null!; - StringBuilder mediumInput = null!; - StringBuilder largeInput = null!; + string smallInput = null!; + string mediumInput = null!; + string largeInput = null!; + + EngineScrubberSet removeEvenSet = null!; + EngineScrubberSet neverMatchesSet = null!; + static Dictionary emptyContext = []; [GlobalSetup] public void Setup() @@ -17,9 +24,12 @@ public void Setup() // Large: ~500KB, 10000 lines largeInput = CreateTestData(10000, 50); + + removeEvenSet = EngineScrubberSet.ForScrubbers([Scrubber.RemoveLines(RemoveEvenLines)]); + neverMatchesSet = EngineScrubberSet.ForScrubbers([Scrubber.RemoveLines(NeverMatches)]); } - static StringBuilder CreateTestData(int lineCount, int charsPerLine) + static string CreateTestData(int lineCount, int charsPerLine) { var builder = new StringBuilder(); for (var i = 0; i < lineCount; i++) @@ -28,7 +38,8 @@ static StringBuilder CreateTestData(int lineCount, int charsPerLine) builder.Append(i); builder.AppendLine(); } - return builder; + + return builder.ToString(); } // Remove every other line @@ -38,45 +49,69 @@ static bool RemoveEvenLines(string line) => // Never matches - no lines removed static bool NeverMatches(string line) => false; + string Engine(EngineScrubberSet set, string content) + { + using var counter = Counter.Start(); + return ScrubEngine.Run(content, set, counter, emptyContext, applyDirectoryReplacements: false); + } + [Benchmark(Baseline = true)] - public void Small() + public void Legacy_Small() { - var builder = new StringBuilder(smallInput.ToString()); + var builder = new StringBuilder(smallInput); builder.FilterLines(RemoveEvenLines); } [Benchmark] - public void Medium() + public void Legacy_Medium() { - var builder = new StringBuilder(mediumInput.ToString()); + var builder = new StringBuilder(mediumInput); builder.FilterLines(RemoveEvenLines); } [Benchmark] - public void Large() + public void Legacy_Large() { - var builder = new StringBuilder(largeInput.ToString()); + var builder = new StringBuilder(largeInput); builder.FilterLines(RemoveEvenLines); } [Benchmark] - public void Small_NoMatches() + public void Legacy_Small_NoMatches() { - var builder = new StringBuilder(smallInput.ToString()); + var builder = new StringBuilder(smallInput); builder.FilterLines(NeverMatches); } [Benchmark] - public void Medium_NoMatches() + public void Legacy_Medium_NoMatches() { - var builder = new StringBuilder(mediumInput.ToString()); + var builder = new StringBuilder(mediumInput); builder.FilterLines(NeverMatches); } [Benchmark] - public void Large_NoMatches() + public void Legacy_Large_NoMatches() { - var builder = new StringBuilder(largeInput.ToString()); + var builder = new StringBuilder(largeInput); builder.FilterLines(NeverMatches); } -} \ No newline at end of file + + [Benchmark] + public string Engine_Small() => Engine(removeEvenSet, smallInput); + + [Benchmark] + public string Engine_Medium() => Engine(removeEvenSet, mediumInput); + + [Benchmark] + public string Engine_Large() => Engine(removeEvenSet, largeInput); + + [Benchmark] + public string Engine_Small_NoMatches() => Engine(neverMatchesSet, smallInput); + + [Benchmark] + public string Engine_Medium_NoMatches() => Engine(neverMatchesSet, mediumInput); + + [Benchmark] + public string Engine_Large_NoMatches() => Engine(neverMatchesSet, largeInput); +} diff --git a/src/Benchmarks/GlobalUsings.cs b/src/Benchmarks/GlobalUsings.cs index 34862b2983..961afb6448 100644 --- a/src/Benchmarks/GlobalUsings.cs +++ b/src/Benchmarks/GlobalUsings.cs @@ -2,3 +2,4 @@ global using BenchmarkDotNet.Attributes; global using BenchmarkDotNet.Running; global using VerifyTests; +global using Culture = System.Globalization.CultureInfo; diff --git a/src/Benchmarks/InlineDateScrubberBenchmarks.cs b/src/Benchmarks/InlineDateScrubberBenchmarks.cs index 8dcf0ec209..92ce1f5476 100644 --- a/src/Benchmarks/InlineDateScrubberBenchmarks.cs +++ b/src/Benchmarks/InlineDateScrubberBenchmarks.cs @@ -1,19 +1,13 @@ -using System.Globalization; - // Calibrated from a scan of 33,707 *.verified.* text files across D:\Code (2026-07). // DateTime placeholders are the single most common scrubber target (14.6% of files, // density p50 6.5 per 1000 chars); DateTimeOffset 2.3%, DateOnly 1.6%. File sizes: // p50=260 chars, p90=2.9KB, p99=31KB. // -// DateScrubber ToString()s the whole builder (via AsSpan) and then tests every window -// of the format's length, so the scan cost dominates and is paid whether or not a date -// is present. A fixed-length format (ISO "yyyy-MM-ddTHH:mm:ss", min==max) takes the -// ReplaceFixedLength path; a variable-length format (short-date "d", min emptyContext = []; + + Action legacyDateTimeScrubber = null!; + Action legacyDateScrubber = null!; + Action legacyDateTimeOffsetScrubber = null!; - Action dateTimeScrubber = null!; - Action dateScrubber = null!; - Action dateTimeOffsetScrubber = null!; + EngineScrubberSet dateTimeSet = null!; + EngineScrubberSet dateSet = null!; + EngineScrubberSet dateTimeOffsetSet = null!; // Plain no-match content shared across the fixed (DateTime) and variable (Date) paths string smallNone = null!; @@ -42,9 +41,13 @@ public class InlineDateScrubberBenchmarks [GlobalSetup] public void Setup() { - dateTimeScrubber = DateScrubber.BuildDateTimeScrubber(dateTimeFormat, null); - dateScrubber = DateScrubber.BuildDateScrubber(dateFormat, null); - dateTimeOffsetScrubber = DateScrubber.BuildDateTimeOffsetScrubber(dateTimeOffsetFormat, null); + legacyDateTimeScrubber = LegacyDateScrubber.BuildDateTimeScrubber(dateTimeFormat, null); + legacyDateScrubber = LegacyDateScrubber.BuildDateScrubber(dateFormat, null); + legacyDateTimeOffsetScrubber = LegacyDateScrubber.BuildDateTimeOffsetScrubber(dateTimeOffsetFormat, null); + + dateTimeSet = EngineScrubberSet.ForScrubbers([.. DateMatchers.DateTimes(dateTimeFormat, null)]); + dateSet = EngineScrubberSet.ForScrubbers([.. DateMatchers.Dates(dateFormat, null)]); + dateTimeOffsetSet = EngineScrubberSet.ForScrubbers([.. DateMatchers.DateTimeOffsets(dateTimeOffsetFormat, null)]); smallNone = Build(260, 0, DateTimeToken); // p50 mediumNone = Build(2_900, 0, DateTimeToken); // p90 @@ -98,33 +101,69 @@ static string Build(int targetChars, int tokenEveryLines, Func toke return builder.ToString(); } + void Legacy(Action scrubber, string content) + { + using var counter = Counter.Start(); + scrubber(new(content), counter); + } + + string Engine(EngineScrubberSet set, string content) + { + using var counter = Counter.Start(); + return ScrubEngine.Run(content, set, counter, emptyContext, applyDirectoryReplacements: false); + } + // DateTime, fixed-length (ISO) path [Benchmark(Baseline = true)] - public void DateTime_Small_None() => dateTimeScrubber(new(smallNone), Counter.Start()); + public void Legacy_DateTime_Small_None() => Legacy(legacyDateTimeScrubber, smallNone); + + [Benchmark] + public void Legacy_DateTime_Medium_None() => Legacy(legacyDateTimeScrubber, mediumNone); + + [Benchmark] + public void Legacy_DateTime_Large_None() => Legacy(legacyDateTimeScrubber, largeNone); [Benchmark] - public void DateTime_Medium_None() => dateTimeScrubber(new(mediumNone), Counter.Start()); + public void Legacy_DateTime_Medium_Typical() => Legacy(legacyDateTimeScrubber, dateTimeMediumTypical); [Benchmark] - public void DateTime_Large_None() => dateTimeScrubber(new(largeNone), Counter.Start()); + public void Legacy_DateTime_Large_Typical() => Legacy(legacyDateTimeScrubber, dateTimeLargeTypical); [Benchmark] - public void DateTime_Medium_Typical() => dateTimeScrubber(new(dateTimeMediumTypical), Counter.Start()); + public string Engine_DateTime_Small_None() => Engine(dateTimeSet, smallNone); [Benchmark] - public void DateTime_Large_Typical() => dateTimeScrubber(new(dateTimeLargeTypical), Counter.Start()); + public string Engine_DateTime_Medium_None() => Engine(dateTimeSet, mediumNone); + + [Benchmark] + public string Engine_DateTime_Large_None() => Engine(dateTimeSet, largeNone); + + [Benchmark] + public string Engine_DateTime_Medium_Typical() => Engine(dateTimeSet, dateTimeMediumTypical); + + [Benchmark] + public string Engine_DateTime_Large_Typical() => Engine(dateTimeSet, dateTimeLargeTypical); // DateOnly, variable-length (short-date) path - same no-match input as DateTime_Medium_None [Benchmark] - public void Date_Medium_None() => dateScrubber(new(mediumNone), Counter.Start()); + public void Legacy_Date_Medium_None() => Legacy(legacyDateScrubber, mediumNone); [Benchmark] - public void Date_Medium_Typical() => dateScrubber(new(dateMediumTypical), Counter.Start()); + public void Legacy_Date_Medium_Typical() => Legacy(legacyDateScrubber, dateMediumTypical); + + [Benchmark] + public string Engine_Date_Medium_None() => Engine(dateSet, mediumNone); + + [Benchmark] + public string Engine_Date_Medium_Typical() => Engine(dateSet, dateMediumTypical); // DateTimeOffset [Benchmark] - public void DateTimeOffset_Medium_Typical() => dateTimeOffsetScrubber(new(dateTimeOffsetMediumTypical), Counter.Start()); + public void Legacy_DateTimeOffset_Medium_Typical() => Legacy(legacyDateTimeOffsetScrubber, dateTimeOffsetMediumTypical); + + [Benchmark] + public string Engine_DateTimeOffset_Medium_Typical() => Engine(dateTimeOffsetSet, dateTimeOffsetMediumTypical); } diff --git a/src/Benchmarks/InlineGuidScrubberBenchmarks.cs b/src/Benchmarks/InlineGuidScrubberBenchmarks.cs index 2577bc5b43..4f4bfaa5be 100644 --- a/src/Benchmarks/InlineGuidScrubberBenchmarks.cs +++ b/src/Benchmarks/InlineGuidScrubberBenchmarks.cs @@ -1,13 +1,10 @@ // Calibrated from a scan of 33,707 *.verified.* text files across D:\Code (2026-07): // size p50=260 chars, p90=2.9KB, p99=31KB. Guid placeholders appear in 5.1% of files // (density p50 1.5, p90 23 per 1000 chars; the densest single file held 1,057). -// GuidScrubber.ReplaceGuids walks StringBuilder.GetChunks() testing every 36-char -// window, so the scan cost is paid on every scrubbed file even when no guid is present -// (the common case). Builders produced by serialization span many chunks and exercise -// the cross-chunk carryover branch (recently bug-fixed); builders from raw-string -// verification are a single chunk. Both shapes are covered. -// A fresh Counter is created per invocation to mirror one-Counter-per-verify; its -// allocation (a handful of empty dictionaries) is a constant floor across all rows. +// Legacy_* rows run the pre-engine chunk-walking implementation (LegacyScrubbers.cs); +// Engine_* rows run the span engine. The Composition rows show the whole-pipeline win: +// guids + dates + line removal as one engine pass vs sequential legacy passes. +// A fresh Counter is created per invocation to mirror one-Counter-per-verify. [MemoryDiagnoser] [SimpleJob(iterationCount: 10, warmupCount: 3)] public class InlineGuidScrubberBenchmarks @@ -18,6 +15,12 @@ public class InlineGuidScrubberBenchmarks string mediumTypical = null!; string largeTypical = null!; string largeDense = null!; + string composition = null!; + + EngineScrubberSet guidSet = null!; + EngineScrubberSet compositionSet = null!; + static Dictionary emptyContext = []; + const string isoFormat = "yyyy-MM-ddTHH:mm:ss"; [GlobalSetup] public void Setup() @@ -28,10 +31,16 @@ public void Setup() mediumTypical = Build(2_900, 12); // ~1 guid / 12 lines ~= p50 density largeTypical = Build(31_000, 12); largeDense = Build(31_000, 1); // a guid on every line - worst case + composition = BuildComposition(31_000); + + guidSet = EngineScrubberSet.ForScrubbers([GuidMatcher.Instance]); + List compositionScrubbers = [GuidMatcher.Instance, Scrubber.RemoveLinesContaining(StringComparison.Ordinal, "SECRET")]; + compositionScrubbers.AddRange(DateMatchers.DateTimes(isoFormat, null)); + compositionSet = EngineScrubberSet.ForScrubbers(compositionScrubbers); } // Lines of ~40 chars; every guidEveryLines'th line embeds a distinct, quote-delimited - // guid in canonical "D" format so GuidScrubber.ReplaceGuids parses and replaces it. + // guid in canonical "D" format. static string Build(int targetChars, int guidEveryLines) { var builder = new StringBuilder(); @@ -61,48 +70,112 @@ static string Build(int targetChars, int guidEveryLines) return builder.ToString(); } - // Rebuilds the same content as many ~200-char chunks, the way an incrementally - // appended serialization buffer arrives, forcing the chunk-carryover branch. - static StringBuilder MultiChunk(string content) + // Mixed content: a guid every 12 lines, an ISO timestamp every 4, a SECRET line every 10 + static string BuildComposition(int targetChars) { var builder = new StringBuilder(); - for (var start = 0; start < content.Length; start += 200) + var line = 0; + var seed = 1; + var baseDate = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Unspecified); + while (builder.Length < targetChars) { - var length = Math.Min(200, content.Length - start); - builder.Append(content.AsSpan(start, length)); + if (line % 12 == 0) + { + builder.Append(" \"id\": \""); + builder.Append($"{seed:x8}-0000-4000-8000-000000000000"); + builder.Append("\","); + } + else if (line % 10 == 0) + { + builder.Append(" [trace] SECRET session token for request "); + builder.Append(line); + } + else if (line % 4 == 0) + { + builder.Append(" \"timestamp\": \""); + builder.Append(baseDate.AddSeconds(seed).ToString(isoFormat, Culture.InvariantCulture)); + builder.Append("\","); + } + else + { + builder.Append(" \"name\": \"item "); + builder.Append(line); + builder.Append(" description text\","); + } + + seed++; + builder.Append('\n'); + line++; } - return builder; + return builder.ToString(); } - static void Scrub(StringBuilder builder) => - GuidScrubber.ReplaceGuids(builder, Counter.Start()); + void LegacyScrub(string content) + { + using var counter = Counter.Start(); + var builder = new StringBuilder(content); + LegacyGuidScrubber.ReplaceGuids(builder, counter); + } - // Single chunk (raw-string verification shape) + string EngineScrub(string content) + { + using var counter = Counter.Start(); + return ScrubEngine.Run(content, guidSet, counter, emptyContext, applyDirectoryReplacements: false); + } [Benchmark(Baseline = true)] - public void Small_None() => Scrub(new(smallNone)); + public void Legacy_Small_None() => LegacyScrub(smallNone); + + [Benchmark] + public void Legacy_Medium_None() => LegacyScrub(mediumNone); + + [Benchmark] + public void Legacy_Large_None() => LegacyScrub(largeNone); + + [Benchmark] + public void Legacy_Medium_Typical() => LegacyScrub(mediumTypical); + + [Benchmark] + public void Legacy_Large_Typical() => LegacyScrub(largeTypical); [Benchmark] - public void Medium_None() => Scrub(new(mediumNone)); + public void Legacy_Large_Dense() => LegacyScrub(largeDense); [Benchmark] - public void Large_None() => Scrub(new(largeNone)); + public string Engine_Small_None() => EngineScrub(smallNone); [Benchmark] - public void Medium_Typical() => Scrub(new(mediumTypical)); + public string Engine_Medium_None() => EngineScrub(mediumNone); [Benchmark] - public void Large_Typical() => Scrub(new(largeTypical)); + public string Engine_Large_None() => EngineScrub(largeNone); [Benchmark] - public void Large_Dense() => Scrub(new(largeDense)); + public string Engine_Medium_Typical() => EngineScrub(mediumTypical); - // Multi chunk (serialized-output shape) - exercises the cross-chunk carryover branch + [Benchmark] + public string Engine_Large_Typical() => EngineScrub(largeTypical); [Benchmark] - public void Large_MultiChunk_None() => Scrub(MultiChunk(largeNone)); + public string Engine_Large_Dense() => EngineScrub(largeDense); + + // Whole-pipeline comparison: three scrubbers over mixed content [Benchmark] - public void Large_MultiChunk_Typical() => Scrub(MultiChunk(largeTypical)); + public void Legacy_Composition() + { + using var counter = Counter.Start(); + var builder = new StringBuilder(composition); + LegacyGuidScrubber.ReplaceGuids(builder, counter); + LegacyDateScrubber.ReplaceDateTimes(builder, isoFormat, counter, Culture.CurrentCulture); + builder.RemoveLinesContaining(StringComparison.Ordinal, "SECRET"); + } + + [Benchmark] + public string Engine_Composition() + { + using var counter = Counter.Start(); + return ScrubEngine.Run(composition, compositionSet, counter, emptyContext, applyDirectoryReplacements: false); + } } diff --git a/src/Benchmarks/LegacyScrubbers.cs b/src/Benchmarks/LegacyScrubbers.cs new file mode 100644 index 0000000000..3a06b276dd --- /dev/null +++ b/src/Benchmarks/LegacyScrubbers.cs @@ -0,0 +1,470 @@ +// Copies of the pre-engine StringBuilder based scrubber implementations, preserved so the +// benchmarks can report before/after rows over identical corpora. Sourced from +// src/Verify/Serialization/Scrubbers (deleted when the span engine replaced them). + +using System.Diagnostics.CodeAnalysis; +using System.Globalization; + +static class LegacyGuidScrubber +{ + public static void ReplaceGuids(StringBuilder builder, Counter counter) + { + if (!counter.ScrubGuids) + { + return; + } + + //{173535ae-995b-4cc6-a74e-8cd4be57039c} + if (builder.Length < 36) + { + return; + } + + var matches = FindMatches(builder, counter); + + // Sort by position descending. In-place to avoid LINQ allocation + matches.Sort((a, b) => b.Index.CompareTo(a.Index)); + + // Apply matches + foreach (var match in matches) + { + builder.Overwrite(match.Value, match.Index, 36); + } + } + + static List FindMatches(StringBuilder builder, Counter counter) + { + var absolutePosition = 0; + var matches = new List(); + Span carryoverBuffer = stackalloc char[35]; + Span buffer = stackalloc char[36]; + var carryoverLength = 0; + var previousChunkAbsoluteEnd = 0; + + foreach (var chunk in builder.GetChunks()) + { + var chunkSpan = chunk.Span; + + // Check for GUIDs spanning from previous chunk to current chunk + if (carryoverLength > 0) + { + // Check each possible starting position in the carryover + for (var carryoverIndex = 0; carryoverIndex < carryoverLength; carryoverIndex++) + { + var remainingInCarryover = carryoverLength - carryoverIndex; + var neededFromCurrent = 36 - remainingInCarryover; + + if (neededFromCurrent <= 0 || + chunkSpan.Length < neededFromCurrent) + { + continue; + } + + carryoverBuffer.Slice(carryoverIndex, remainingInCarryover).CopyTo(buffer); + chunkSpan[..neededFromCurrent].CopyTo(buffer[remainingInCarryover..]); + + // Check boundary characters + var startPosition = previousChunkAbsoluteEnd - carryoverLength + carryoverIndex; + + var hasValidStart = startPosition == 0 || + !IsInvalidStartingChar(builder[startPosition - 1]); + + if (!hasValidStart) + { + continue; + } + + var hasValidEnd = neededFromCurrent >= chunkSpan.Length || + !IsInvalidEndingChar(chunkSpan[neededFromCurrent]); + + if (!hasValidEnd) + { + continue; + } + + if (!Guid.TryParseExact(buffer, "D", out var guid)) + { + continue; + } + + var convert = counter.Convert(guid); + matches.Add(new(startPosition, convert)); + } + } + + // Process GUIDs entirely within this chunk + if (chunk.Length >= 36) + { + for (var chunkIndex = 0; chunkIndex < chunk.Length; chunkIndex++) + { + var end = chunkIndex + 36; + if (end > chunk.Length) + { + break; + } + + var value = chunkSpan; + if ((chunkIndex != 0 && IsInvalidStartingChar(value[chunkIndex - 1])) || + (end != value.Length && IsInvalidEndingChar(value[end]))) + { + continue; + } + + var slice = value.Slice(chunkIndex, 36); + + if (!Guid.TryParseExact(slice, "D", out var guid)) + { + continue; + } + + var convert = counter.Convert(guid); + var startReplaceIndex = absolutePosition + chunkIndex; + matches.Add(new(startReplaceIndex, convert)); + chunkIndex += 35; + } + } + + // Roll the carryover forward: keep the last 35 chars of everything seen so far + if (chunk.Length >= 35) + { + chunkSpan.Slice(chunk.Length - 35, 35).CopyTo(carryoverBuffer); + carryoverLength = 35; + } + else + { + var keep = Math.Min(carryoverLength, 35 - chunk.Length); + carryoverBuffer.Slice(carryoverLength - keep, keep).CopyTo(carryoverBuffer); + chunkSpan.CopyTo(carryoverBuffer[keep..]); + carryoverLength = keep + chunk.Length; + } + + previousChunkAbsoluteEnd = absolutePosition + chunk.Length; + absolutePosition += chunk.Length; + } + + return matches; + } + + static bool IsInvalidEndingChar(char ch) => + IsInvalidChar(ch) && + ch != '}' && + ch != ')'; + + static bool IsInvalidChar(char ch) => + char.IsLetter(ch) || + char.IsNumber(ch); + + static bool IsInvalidStartingChar(char ch) => + IsInvalidChar(ch) && + ch != '{' && + ch != '('; + + readonly struct Match(int index, string value) + { + public readonly int Index = index; + public readonly string Value = value; + } +} + +static class LegacyDateScrubber +{ + delegate bool TryConvert( + ReadOnlySpan span, + string format, + Counter counter, + CultureInfo culture, + [NotNullWhen(true)] out string? result); + + static bool TryConvertDate( + ReadOnlySpan span, + string format, + Counter counter, + CultureInfo culture, + [NotNullWhen(true)] out string? result) + { + if (DateOnly.TryParseExact(span, format, culture, DateTimeStyles.None, out var date)) + { + result = counter.Convert(date); + return true; + } + + result = null; + return false; + } + + public static Action BuildDateScrubber(string format, CultureInfo? culture) => + (builder, counter) => ReplaceDates(builder, format, counter, culture ?? CultureInfo.CurrentCulture); + + public static void ReplaceDates(StringBuilder builder, string format, Counter counter, CultureInfo culture) => + ReplaceInner( + builder, + format, + counter, + culture, + TryConvertDate); + + public static Action BuildDateTimeOffsetScrubber(string format, CultureInfo? culture) => + (builder, counter) => ReplaceDateTimeOffsets(builder, format, counter, culture ?? CultureInfo.CurrentCulture); + + static bool TryConvertDateTimeOffset( + ReadOnlySpan span, + string format, + Counter counter, + CultureInfo culture, + [NotNullWhen(true)] out string? result) + { + if (DateTimeOffset.TryParseExact(span, format, culture, DateTimeStyles.None, out var date)) + { + result = counter.Convert(date); + return true; + } + + result = null; + return false; + } + + public static void ReplaceDateTimeOffsets(StringBuilder builder, string format, Counter counter, CultureInfo culture) + { + ReplaceInner( + builder, + format, + counter, + culture, + TryConvertDateTimeOffset); + if (TryGetFormatWithUpperMillisecondsTrimmed(format, out var trimmedFormat)) + { + ReplaceInner( + builder, + trimmedFormat, + counter, + culture, + TryConvertDateTimeOffset); + } + } + + static bool TryConvertDateTime( + ReadOnlySpan span, + string format, + Counter counter, + CultureInfo culture, + [NotNullWhen(true)] out string? result) + { + if (DateTime.TryParseExact(span, format, culture, DateTimeStyles.None, out var date)) + { + result = counter.Convert(date); + return true; + } + + result = null; + return false; + } + + public static Action BuildDateTimeScrubber(string format, CultureInfo? culture) => + (builder, counter) => ReplaceDateTimes(builder, format, counter, culture ?? CultureInfo.CurrentCulture); + + public static void ReplaceDateTimes(StringBuilder builder, string format, Counter counter, CultureInfo culture) + { + ReplaceInner( + builder, + format, + counter, + culture, + TryConvertDateTime); + if (TryGetFormatWithUpperMillisecondsTrimmed(format, out var trimmedFormat)) + { + ReplaceInner( + builder, + trimmedFormat, + counter, + culture, + TryConvertDateTime); + } + } + + static bool TryGetFormatWithUpperMillisecondsTrimmed(string format, [NotNullWhen(true)] out string? trimmedFormat) + { + if (format.EndsWith(".FFFF")) + { + trimmedFormat = format[..^5]; + return true; + } + + if (format.EndsWith(".FFF")) + { + trimmedFormat = format[..^4]; + return true; + } + + if (format.EndsWith(".FF")) + { + trimmedFormat = format[..^3]; + return true; + } + + if (format.EndsWith(".F")) + { + trimmedFormat = format[..^2]; + return true; + } + + trimmedFormat = null; + return false; + } + + static void ReplaceInner(StringBuilder builder, string format, Counter counter, CultureInfo culture, TryConvert tryConvertDate) + { + if (!counter.ScrubDateTimes) + { + return; + } + + var (max, min) = DateFormatLengthCalculator.GetLength(format, culture); + + if (builder.Length < min) + { + return; + } + + if (min == max) + { + ReplaceFixedLength(builder, format, counter, culture, tryConvertDate, max); + + return; + } + + ReplaceVariableLength(builder, format, counter, culture, tryConvertDate, max, min); + } + + static void ReplaceVariableLength(StringBuilder builder, string format, Counter counter, CultureInfo culture, TryConvert tryConvertDate, int longest, int shortest) + { + var value = builder.AsSpan(); + var builderIndex = 0; + for (var index = 0; index <= value.Length; index++) + { + var found = false; + for (var length = longest; length >= shortest; length--) + { + var end = index + length; + if (end > value.Length) + { + continue; + } + + var slice = value.Slice(index, length); + if (tryConvertDate(slice, format, counter, culture, out var convert)) + { + builder.Overwrite(convert, builderIndex, length); + builderIndex += convert.Length; + index += length - 1; + found = true; + break; + } + } + + if (found) + { + continue; + } + + builderIndex++; + } + } + + static void ReplaceFixedLength(StringBuilder builder, string format, Counter counter, CultureInfo culture, TryConvert tryConvertDate, int length) + { + var value = builder.AsSpan(); + var builderIndex = 0; + var increment = length - 1; + for (var index = 0; index <= value.Length - length; index++) + { + var slice = value.Slice(index, length); + if (tryConvertDate(slice, format, counter, culture, out var convert)) + { + builder.Overwrite(convert, builderIndex, length); + builderIndex += convert.Length; + index += increment; + } + else + { + builderIndex++; + } + } + } +} + +static class LegacyLineScrubber +{ + public static void FilterLines(this StringBuilder input, Func removeLine) + { + var theString = input.ToString(); + using var reader = new StringReader(theString); + input.Clear(); + + while (reader.ReadLine() is { } line) + { + if (removeLine(line)) + { + continue; + } + + input.AppendLineN(line); + } + + var endsWithNewLine = theString.EndsWith('\n'); + if (input.Length > 0 && !endsWithNewLine) + { + input.Length -= 1; + } + } + + public static void RemoveEmptyLines(this StringBuilder builder) + { + builder.FilterLines(string.IsNullOrWhiteSpace); + if (builder.Length > 0 && + builder[0] is '\n') + { + builder.Remove(0, 1); + } + + if (builder.Length > 0 && + builder[^1] is '\n') + { + builder.Length--; + } + } + + public static void RemoveLinesContaining(this StringBuilder input, StringComparison comparison, params string[] stringToMatch) => + input.FilterLines(_ => + { + foreach (var toMatch in stringToMatch) + { + if (_.Contains(toMatch, comparison)) + { + return true; + } + } + + return false; + }); + + public static void ReplaceLines(this StringBuilder input, Func replaceLine) + { + var theString = input.ToString(); + using var reader = new StringReader(theString); + input.Clear(); + while (reader.ReadLine() is { } line) + { + var value = replaceLine(line); + if (value is not null) + { + input.AppendLineN(value); + } + } + + if (input.Length > 0 && + !theString.EndsWith('\n')) + { + input.Length -= 1; + } + } +} diff --git a/src/Benchmarks/LineScrubberBenchmarks.cs b/src/Benchmarks/LineScrubberBenchmarks.cs index 5c9d161124..45fdf7237c 100644 --- a/src/Benchmarks/LineScrubberBenchmarks.cs +++ b/src/Benchmarks/LineScrubberBenchmarks.cs @@ -1,10 +1,9 @@ // Calibrated from a scan of 33,707 *.verified.* text files across D:\Code (2026-07): -// line counts p50=15, p90=88, p99=508. The line scrubbers round-trip the whole builder -// through StringReader.ReadLine (one string allocation per line) and rebuild it, so cost -// and allocations scale with line count rather than raw byte size. -// FilterLines (the predicate path used by ScrubLines) is covered by FilterLinesBenchmarks; -// this covers the Contains-matching (ScrubLinesContaining), empty-line (ScrubEmptyLines) -// and replace (ScrubLinesWithReplace) paths. +// line counts p50=15, p90=88, p99=508. ScrubLinesContaining is the most common hand-written +// scrubber in the wild (56 call sites over 15 repos, 1-2 needles, OrdinalIgnoreCase default). +// Legacy_* rows run the pre-engine implementation: whole-document ToString + StringReader +// (one string allocation per line) + rebuild. Engine_* rows run the span engine line phase +// (per-line span checks; kept lines never allocate). [MemoryDiagnoser] [SimpleJob(iterationCount: 10, warmupCount: 3)] public class LineScrubberBenchmarks @@ -14,6 +13,12 @@ public class LineScrubberBenchmarks string large = null!; string mediumWithBlanks = null!; + EngineScrubberSet containsOrdinalSet = null!; + EngineScrubberSet containsIgnoreCaseSet = null!; + EngineScrubberSet emptyLinesSet = null!; + EngineScrubberSet replaceLinesSet = null!; + static Dictionary emptyContext = []; + [GlobalSetup] public void Setup() { @@ -21,6 +26,11 @@ public void Setup() medium = Build(88, false); // p90 large = Build(508, false); // p99 mediumWithBlanks = Build(88, true); + + containsOrdinalSet = EngineScrubberSet.ForScrubbers([Scrubber.RemoveLinesContaining(StringComparison.Ordinal, "SECRET")]); + containsIgnoreCaseSet = EngineScrubberSet.ForScrubbers([Scrubber.RemoveLinesContaining("secret")]); + emptyLinesSet = EngineScrubberSet.ForScrubbers([Scrubber.RemoveEmptyLines()]); + replaceLinesSet = EngineScrubberSet.ForScrubbers([Scrubber.ReplaceLines(Keep)]); } // Lines of ~55 chars; every 10th line carries a "SECRET" marker (the kind of noise @@ -58,29 +68,65 @@ static string Build(int lineCount, bool withBlanks) static string? Keep(string line) => line; + string Engine(EngineScrubberSet set, string content) + { + using var counter = Counter.Start(); + return ScrubEngine.Run(content, set, counter, emptyContext, applyDirectoryReplacements: false); + } + // ScrubLinesContaining - Contains match on every line [Benchmark(Baseline = true)] - public void RemoveLinesContaining_Small() => + public void Legacy_RemoveLinesContaining_Small() => new StringBuilder(small).RemoveLinesContaining(StringComparison.Ordinal, "SECRET"); [Benchmark] - public void RemoveLinesContaining_Medium() => + public void Legacy_RemoveLinesContaining_Medium() => new StringBuilder(medium).RemoveLinesContaining(StringComparison.Ordinal, "SECRET"); [Benchmark] - public void RemoveLinesContaining_Large() => + public void Legacy_RemoveLinesContaining_Large() => new StringBuilder(large).RemoveLinesContaining(StringComparison.Ordinal, "SECRET"); + [Benchmark] + public string Engine_RemoveLinesContaining_Small() => + Engine(containsOrdinalSet, small); + + [Benchmark] + public string Engine_RemoveLinesContaining_Medium() => + Engine(containsOrdinalSet, medium); + + [Benchmark] + public string Engine_RemoveLinesContaining_Large() => + Engine(containsOrdinalSet, large); + + // OrdinalIgnoreCase is the wild default (44 of 56 call sites) + + [Benchmark] + public void Legacy_RemoveLinesContaining_Large_IgnoreCase() => + new StringBuilder(large).RemoveLinesContaining(StringComparison.OrdinalIgnoreCase, "secret"); + + [Benchmark] + public string Engine_RemoveLinesContaining_Large_IgnoreCase() => + Engine(containsIgnoreCaseSet, large); + // ScrubEmptyLines [Benchmark] - public void RemoveEmptyLines_Medium() => + public void Legacy_RemoveEmptyLines_Medium() => new StringBuilder(mediumWithBlanks).RemoveEmptyLines(); + [Benchmark] + public string Engine_RemoveEmptyLines_Medium() => + Engine(emptyLinesSet, mediumWithBlanks); + // ScrubLinesWithReplace - identity replace isolates the read/rewrite round-trip [Benchmark] - public void ReplaceLines_Medium() => + public void Legacy_ReplaceLines_Medium() => new StringBuilder(medium).ReplaceLines(Keep); + + [Benchmark] + public string Engine_ReplaceLines_Medium() => + Engine(replaceLinesSet, medium); } diff --git a/src/StaticSettingsTests/UserMachineScrubberTests.cs b/src/StaticSettingsTests/UserMachineScrubberTests.cs index 2133fb5bb8..8e6ac57ad1 100644 --- a/src/StaticSettingsTests/UserMachineScrubberTests.cs +++ b/src/StaticSettingsTests/UserMachineScrubberTests.cs @@ -5,13 +5,12 @@ public UserMachineScrubberTests() => UserMachineScrubber.ResetReplacements("FakeMachineName", "FakeUserName"); [Fact] - public void MultipleChunks() + public void Replace() { - var builder = new StringBuilder(capacity: 8, maxCapacity: int.MaxValue); - builder.Append("FakeUserName"); using var counter = Counter.Start(); - UserMachineScrubber.PerformReplacements(builder, "FakeUserName", "TheUserName"); - Assert.Equal("TheUserName", builder.ToString()); + var set = EngineScrubberSet.ForScrubbers([UserMachineScrubber.UserScrubber()]); + var result = ScrubEngine.Run("FakeUserName", set, counter, new Dictionary(), applyDirectoryReplacements: false); + Assert.Equal("TheUserName", result); } [Fact] diff --git a/src/Verify.Tests/DateScrubberTests.cs b/src/Verify.Tests/DateScrubberTests.cs index 223b1903a5..2251ccb28b 100644 --- a/src/Verify.Tests/DateScrubberTests.cs +++ b/src/Verify.Tests/DateScrubberTests.cs @@ -19,6 +19,14 @@ public static void Init() public Task GetCultureDates() => Verify(DateFormatLengthCalculator.GetCultureLengthInfo(CultureInfo.InvariantCulture)); + static Dictionary emptyContext = []; + + static string Scrub(string value, Scrubber[] scrubbers, Counter counter) + { + var set = EngineScrubberSet.ForScrubbers([.. scrubbers]); + return ScrubEngine.Run(value, set, counter, emptyContext, applyDirectoryReplacements: false); + } + [Theory] [InlineData("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "no match")] [InlineData("aaaa", "no match short")] @@ -31,9 +39,8 @@ public Task GetCultureDates() => public async Task DateTimeOffsets(string value, string name) { using var counter = Counter.Start(); - var builder = new StringBuilder(value); - DateScrubber.ReplaceDateTimeOffsets(builder, "yyyy-MM-dd", counter, CultureInfo.InvariantCulture); - await Verify(builder) + var result = Scrub(value, DateMatchers.DateTimeOffsets("yyyy-MM-dd", CultureInfo.InvariantCulture), counter); + await Verify(result) .UseTextForParameters(name); } @@ -49,9 +56,8 @@ await Verify(builder) public async Task VariableLengthDateTimeOffsets(string value, string name) { using var counter = Counter.Start(); - var builder = new StringBuilder(value); - DateScrubber.ReplaceDateTimeOffsets(builder, "yyyy MMMM dd dddd", counter, CultureInfo.InvariantCulture); - await Verify(builder) + var result = Scrub(value, DateMatchers.DateTimeOffsets("yyyy MMMM dd dddd", CultureInfo.InvariantCulture), counter); + await Verify(result) .UseTextForParameters(name); } @@ -79,9 +85,8 @@ public Task NamedDateTimeOffsets(string value, string name) => public async Task DateTimes(string value, string name) { using var counter = Counter.Start(); - var builder = new StringBuilder(value); - DateScrubber.ReplaceDateTimes(builder, "yyyy-MM-dd", counter, CultureInfo.InvariantCulture); - await Verify(builder) + var result = Scrub(value, DateMatchers.DateTimes("yyyy-MM-dd", CultureInfo.InvariantCulture), counter); + await Verify(result) .UseTextForParameters(name); } @@ -91,7 +96,7 @@ public void ReplaceDateTimes_AllCultures() foreach (var culture in CultureInfo.GetCultures(CultureTypes.AllCultures)) { var format = "yyyy MMMM MMM MM dddd ddd dd d HH H mm m ss s fffff tt"; - var counter = Counter.Start(); + using var counter = Counter.Start(); var dateTime = DateTime.Now; var dateFormat = culture.DateTimeFormat; @@ -102,9 +107,7 @@ public void ReplaceDateTimes_AllCultures() } var value = dateTime.ToString(format, culture); - var builder = new StringBuilder(value); - DateScrubber.ReplaceDateTimes(builder, format, counter, culture); - var result = builder.ToString(); + var result = Scrub(value, DateMatchers.DateTimes(format, culture), counter); if (result == "DateTime_1") { continue; @@ -131,9 +134,8 @@ public void ReplaceDateTimes_AllCultures() public async Task VariableLengthDateTimes(string value, string name) { using var counter = Counter.Start(); - var builder = new StringBuilder(value); - DateScrubber.ReplaceDateTimes(builder, "yyyy MMMM dd dddd", counter, CultureInfo.InvariantCulture); - await Verify(builder) + var result = Scrub(value, DateMatchers.DateTimes("yyyy MMMM dd dddd", CultureInfo.InvariantCulture), counter); + await Verify(result) .UseTextForParameters(name); } @@ -162,9 +164,8 @@ public Task NamedDateTimes(string value, string name) => public async Task Dates(string value, string name) { using var counter = Counter.Start(); - var builder = new StringBuilder(value); - DateScrubber.ReplaceDates(builder, "yyyy-MM-dd", counter, CultureInfo.InvariantCulture); - await Verify(builder) + var result = Scrub(value, DateMatchers.Dates("yyyy-MM-dd", CultureInfo.InvariantCulture), counter); + await Verify(result) .UseTextForParameters(name); } @@ -180,9 +181,8 @@ await Verify(builder) public async Task VariableLengthDates(string value, string name) { using var counter = Counter.Start(); - var builder = new StringBuilder(value); - DateScrubber.ReplaceDates(builder, "yyyy MMMM dd dddd", counter, CultureInfo.InvariantCulture); - await Verify(builder) + var result = Scrub(value, DateMatchers.Dates("yyyy MMMM dd dddd", CultureInfo.InvariantCulture), counter); + await Verify(result) .UseTextForParameters(name); } diff --git a/src/Verify.Tests/GuidScrubberTests.cs b/src/Verify.Tests/GuidScrubberTests.cs index 531b80d042..3c179b8da2 100644 --- a/src/Verify.Tests/GuidScrubberTests.cs +++ b/src/Verify.Tests/GuidScrubberTests.cs @@ -7,6 +7,14 @@ public class GuidScrubberTests #endregion + static Dictionary emptyContext = []; + + static string ReplaceGuids(string value, Counter counter) + { + var set = EngineScrubberSet.ForScrubbers([GuidMatcher.Instance]); + return ScrubEngine.Run(value, set, counter, emptyContext, applyDirectoryReplacements: false); + } + [Theory] [InlineData("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "no match")] [InlineData("aaaaaaaaaaaaaaaaaaaaaaaa", "no match short")] @@ -43,9 +51,8 @@ public class GuidScrubberTests public async Task Run(string guid, string name) { using var counter = Counter.Start(); - var builder = new StringBuilder(guid); - GuidScrubber.ReplaceGuids(builder, counter); - await Verify(builder) + var result = ReplaceGuids(guid, counter); + await Verify(result) .DontScrubGuids() .UseTextForParameters(name); } @@ -91,31 +98,11 @@ public Task NamedGuidTopLevelInstance() } [Fact] - public void MultipleChunks() - { - var builder = new StringBuilder(capacity: 8, maxCapacity: int.MaxValue); - builder.Append("[2e6bddf7-fcf7-4b09-bb6f-a7948e1eecf3]"); - builder.Append("[c2eeaf99-d5c4-4341-8543-4597c3fd40d9]"); - using var counter = Counter.Start(); - GuidScrubber.ReplaceGuids(builder, counter); - Assert.Equal("[Guid_1][Guid_2]", builder.ToString()); - } - - [Fact] - public void GuidSpanningThreeChunks() + public void Multiple() { - // Capacity 4 forces small chunks [4][4][rest]; the 36-char guid spans all - // three, and the 4-char middle chunk is shorter than the 35-char carryover. - // The carryover must accumulate across chunks or the prefix is dropped and - // the guid is never found (silent leak). - var guid = "173535ae-995b-4cc6-a74e-8cd4be57039c"; - var builder = new StringBuilder(capacity: 4); - builder.Append(guid[..4]); // chunk0 - builder.Append(guid[4..8]); // chunk1 (short middle chunk) - builder.Append(guid[8..]); // chunk2 using var counter = Counter.Start(); - GuidScrubber.ReplaceGuids(builder, counter); - Assert.Equal("Guid_1", builder.ToString()); + var result = ReplaceGuids("[2e6bddf7-fcf7-4b09-bb6f-a7948e1eecf3][c2eeaf99-d5c4-4341-8543-4597c3fd40d9]", counter); + Assert.Equal("[Guid_1][Guid_2]", result); } #region NamedGuidFluent @@ -161,4 +148,4 @@ public Task NamedGuidTopLevelFluent() [Fact] public Task NamedGuidTopLevelGlobal() => Verify(new Guid("c8eeaf99-d5c4-4341-8543-4597c3fd40c9")); -} \ No newline at end of file +} diff --git a/src/Verify.Tests/LinesScrubberTests.cs b/src/Verify.Tests/LinesScrubberTests.cs index 8bf274259e..4aaa6f82a7 100644 --- a/src/Verify.Tests/LinesScrubberTests.cs +++ b/src/Verify.Tests/LinesScrubberTests.cs @@ -1,4 +1,4 @@ -public class LinesScrubberTests +public class LinesScrubberTests { [Fact] public Task ScrubLinesContaining() @@ -78,317 +78,158 @@ public Task ScrubLinesContaining_case_sensitive() """); } - [Fact] - public void ReplaceLines_AllRemovedNoTrailingNewline() - { - // Single line, no trailing newline, and every line replaced with null. - // The trailing-newline trim must guard on the rebuilt builder length, not - // the original string length, otherwise Length -= 1 underflows. - var builder = new StringBuilder("single line"); - - builder.ReplaceLines(_ => null); - - Assert.Equal(string.Empty, builder.ToString()); - } + static Dictionary emptyContext = []; - [Fact] - public void ReplaceLines_ReplacesLines() + static string Run(string input, Scrubber scrubber) { - var builder = new StringBuilder("a\nb\nc"); - - builder.ReplaceLines(line => line == "b" ? "B" : line); - - Assert.Equal("a\nB\nc", builder.ToString()); + using var counter = Counter.Start(); + var set = EngineScrubberSet.ForScrubbers([scrubber]); + return ScrubEngine.Run(input, set, counter, emptyContext, applyDirectoryReplacements: false); } - [Fact] - public void FilterLines_RemovesSingleLine() - { - var builder = new StringBuilder("line1\nline2\nline3"); + static string Filter(string input, Func removeLine) => + Run(input, Scrubber.RemoveLines(removeLine)); - builder.FilterLines(line => line == "line2"); - - Assert.Equal("line1\nline3", builder.ToString()); - } + static string Replace(string input, Func replaceLine) => + Run(input, Scrubber.ReplaceLines(replaceLine)); [Fact] - public void FilterLines_RemovesMultipleLines() - { - var builder = new StringBuilder("keep1\nremove1\nkeep2\nremove2\nkeep3"); - - builder.FilterLines(line => line.StartsWith("remove")); - - Assert.Equal("keep1\nkeep2\nkeep3", builder.ToString()); - } + public void ReplaceLines_AllRemovedNoTrailingNewline() => + // Single line, no trailing newline, and every line replaced with null + Assert.Equal(string.Empty, Replace("single line", _ => null)); [Fact] - public void FilterLines_RemovesAllLines() - { - var builder = new StringBuilder("line1\nline2\nline3"); - - builder.FilterLines(_ => true); - - Assert.Equal(string.Empty, builder.ToString()); - } + public void ReplaceLines_ReplacesLines() => + Assert.Equal("a\nB\nc", Replace("a\nb\nc", _ => _ == "b" ? "B" : _)); [Fact] - public void FilterLines_RemovesNoLines() - { - var builder = new StringBuilder("line1\nline2\nline3"); - var original = builder.ToString(); - - builder.FilterLines(_ => false); - - Assert.Equal(original, builder.ToString()); - } + public void FilterLines_RemovesSingleLine() => + Assert.Equal("line1\nline3", Filter("line1\nline2\nline3", _ => _ == "line2")); [Fact] - public void FilterLines_EmptyStringBuilder() - { - var builder = new StringBuilder(); - - builder.FilterLines(_ => true); - - Assert.Equal(string.Empty, builder.ToString()); - } + public void FilterLines_RemovesMultipleLines() => + Assert.Equal("keep1\nkeep2\nkeep3", Filter("keep1\nremove1\nkeep2\nremove2\nkeep3", _ => _.StartsWith("remove"))); [Fact] - public void FilterLines_SingleLineNoNewline() - { - var builder = new StringBuilder("single line"); - - builder.FilterLines(_ => false); - - Assert.Equal("single line", builder.ToString()); - } + public void FilterLines_RemovesAllLines() => + Assert.Equal(string.Empty, Filter("line1\nline2\nline3", _ => true)); [Fact] - public void FilterLines_SingleLineWithNewline() - { - var builder = new StringBuilder("single line\n"); - - builder.FilterLines(_ => false); - - Assert.Equal("single line\n", builder.ToString()); - } + public void FilterLines_RemovesNoLines() => + Assert.Equal("line1\nline2\nline3", Filter("line1\nline2\nline3", _ => false)); [Fact] - public void FilterLines_PreservesTrailingNewline() - { - var builder = new StringBuilder("line1\nline2\n"); - - builder.FilterLines(line => line == "line2"); - - Assert.Equal("line1\n", builder.ToString()); - } + public void FilterLines_EmptyString() => + Assert.Equal(string.Empty, Filter(string.Empty, _ => true)); [Fact] - public void FilterLines_DoesNotAddTrailingNewlineWhenOriginalLacked() - { - var builder = new StringBuilder("line1\nline2"); - - builder.FilterLines(line => line == "line1"); - - Assert.Equal("line2", builder.ToString()); - } + public void FilterLines_SingleLineNoNewline() => + Assert.Equal("single line", Filter("single line", _ => false)); [Fact] - public void FilterLines_HandlesEmptyLines() - { - var builder = new StringBuilder("line1\n\nline3"); - - builder.FilterLines(string.IsNullOrEmpty); - - Assert.Equal("line1\nline3", builder.ToString()); - } + public void FilterLines_SingleLineWithNewline() => + Assert.Equal("single line\n", Filter("single line\n", _ => false)); [Fact] - public void FilterLines_KeepsEmptyLines() - { - var builder = new StringBuilder("line1\n\nline3"); - - builder.FilterLines(line => line == "line1"); - - Assert.Equal("\nline3", builder.ToString()); - } + public void FilterLines_PreservesTrailingNewline() => + Assert.Equal("line1\n", Filter("line1\nline2\n", _ => _ == "line2")); [Fact] - public void FilterLines_OnlyEmptyLines() - { - var builder = new StringBuilder("\n\n"); - - builder.FilterLines(_ => false); - - Assert.Equal("\n\n", builder.ToString()); - } + public void FilterLines_DoesNotAddTrailingNewlineWhenOriginalLacked() => + Assert.Equal("line2", Filter("line1\nline2", _ => _ == "line1")); [Fact] - public void FilterLines_RemovesFirstLine() - { - var builder = new StringBuilder("remove\nkeep1\nkeep2"); - - builder.FilterLines(line => line == "remove"); - - Assert.Equal("keep1\nkeep2", builder.ToString()); - } + public void FilterLines_HandlesEmptyLines() => + Assert.Equal("line1\nline3", Filter("line1\n\nline3", string.IsNullOrEmpty)); [Fact] - public void FilterLines_RemovesLastLine() - { - var builder = new StringBuilder("keep1\nkeep2\nremove"); - - builder.FilterLines(line => line == "remove"); - - Assert.Equal("keep1\nkeep2", builder.ToString()); - } + public void FilterLines_KeepsEmptyLines() => + Assert.Equal("\nline3", Filter("line1\n\nline3", _ => _ == "line1")); [Fact] - public void FilterLines_RemovesLastLineWithTrailingNewline() - { - var builder = new StringBuilder("keep1\nkeep2\nremove\n"); - - builder.FilterLines(line => line == "remove"); - - Assert.Equal("keep1\nkeep2\n", builder.ToString()); - } + public void FilterLines_OnlyEmptyLines() => + Assert.Equal("\n\n", Filter("\n\n", _ => false)); [Fact] - public void FilterLines_ComplexPredicate() - { - var builder = new StringBuilder("abc123\ndef456\nghi789\njkl012"); - - builder.FilterLines(line => line.Any(char.IsDigit) && line.Contains('4')); - - Assert.Equal("abc123\nghi789\njkl012", builder.ToString()); - } + public void FilterLines_RemovesFirstLine() => + Assert.Equal("keep1\nkeep2", Filter("remove\nkeep1\nkeep2", _ => _ == "remove")); [Fact] - public void FilterLines_WindowsLineEndings() - { - var builder = new StringBuilder("line1\r\nline2\r\nline3"); - - builder.FilterLines(line => line == "line2"); + public void FilterLines_RemovesLastLine() => + Assert.Equal("keep1\nkeep2", Filter("keep1\nkeep2\nremove", _ => _ == "remove")); - // Note: StringReader normalizes \r\n to \n - Assert.Equal("line1\nline3", builder.ToString()); - } + [Fact] + public void FilterLines_RemovesLastLineWithTrailingNewline() => + Assert.Equal("keep1\nkeep2\n", Filter("keep1\nkeep2\nremove\n", _ => _ == "remove")); [Fact] - public void FilterLines_MixedLineEndings() - { - var builder = new StringBuilder("line1\nline2\r\nline3"); + public void FilterLines_ComplexPredicate() => + Assert.Equal("abc123\nghi789\njkl012", Filter("abc123\ndef456\nghi789\njkl012", _ => _.Any(char.IsDigit) && _.Contains('4'))); - builder.FilterLines(line => line == "line2"); + [Fact] + public void FilterLines_WindowsLineEndings() => + // \r\n is normalized to \n + Assert.Equal("line1\nline3", Filter("line1\r\nline2\r\nline3", _ => _ == "line2")); - Assert.Equal("line1\nline3", builder.ToString()); - } + [Fact] + public void FilterLines_MixedLineEndings() => + Assert.Equal("line1\nline3", Filter("line1\nline2\r\nline3", _ => _ == "line2")); [Fact] public void FilterLines_LargeContent() { - var lines = Enumerable.Range(1, 1000).Select(i => $"line{i}"); - var builder = new StringBuilder(string.Join('\n', lines)); + var lines = Enumerable.Range(1, 1000).Select(_ => $"line{_}"); + var input = string.Join('\n', lines); - builder.FilterLines(line => int.Parse(line[4..]) % 2 == 0); + var result = Filter(input, _ => int.Parse(_[4..]) % 2 == 0); - var remaining = builder.ToString().Split('\n'); + var remaining = result.Split('\n'); Assert.Equal(500, remaining.Length); - Assert.All(remaining, line => Assert.True(int.Parse(line[4..]) % 2 == 1)); + Assert.All(remaining, _ => Assert.True(int.Parse(_[4..]) % 2 == 1)); } [Fact] - public void FilterLines_PreservesWhitespace() - { - var builder = new StringBuilder(" line1 \n\t line2\t\nline3 "); - - builder.FilterLines(line => line.Trim() == "line2"); - - Assert.Equal(" line1 \nline3 ", builder.ToString()); - } + public void FilterLines_PreservesWhitespace() => + Assert.Equal(" line1 \nline3 ", Filter(" line1 \n\t line2\t\nline3 ", _ => _.Trim() == "line2")); [Fact] - public void FilterLines_SingleLineRemoved() - { - var builder = new StringBuilder("remove this"); - - builder.FilterLines(_ => true); - - Assert.Equal(string.Empty, builder.ToString()); - } + public void FilterLines_SingleLineRemoved() => + Assert.Equal(string.Empty, Filter("remove this", _ => true)); [Fact] - public void FilterLines_AllButLastRemoved() - { - var builder = new StringBuilder("remove1\nremove2\nkeep\n"); - - builder.FilterLines(line => line.StartsWith("remove")); - - Assert.Equal("keep\n", builder.ToString()); - } + public void FilterLines_AllButLastRemoved() => + Assert.Equal("keep\n", Filter("remove1\nremove2\nkeep\n", _ => _.StartsWith("remove"))); [Fact] - public void FilterLines_MultipleConsecutiveRemoved() - { - var builder = new StringBuilder("keep1\nremove1\nremove2\nremove3\nkeep2"); - - builder.FilterLines(line => line.StartsWith("remove")); - - Assert.Equal("keep1\nkeep2", builder.ToString()); - } + public void FilterLines_MultipleConsecutiveRemoved() => + Assert.Equal("keep1\nkeep2", Filter("keep1\nremove1\nremove2\nremove3\nkeep2", _ => _.StartsWith("remove"))); [Fact] - public void FilterLines_LineSpansMultipleChunks() + public void FilterLines_VeryLongLine() { - // StringBuilder default chunk size is ~8000 chars - // Create a line that's definitely longer than a chunk var longLine = new string('a', 10_000); - var builder = new StringBuilder(); - builder.Append("keep1\n"); - builder.Append(longLine); - builder.Append("\nkeep2"); - - builder.FilterLines(line => line == longLine); - Assert.Equal("keep1\nkeep2", builder.ToString()); - } + var result = Filter($"keep1\n{longLine}\nkeep2", _ => _ == longLine); - [Fact] - public void FilterLines_NewlineAtChunkBoundary() - { - // Fill first chunk, then add newline at boundary - var chunkFiller = new string('x', 8000); - var builder = new StringBuilder(); - builder.Append(chunkFiller); - builder.Append('\n'); - builder.Append("line2"); - - builder.FilterLines(line => line == chunkFiller); - - Assert.Equal("line2", builder.ToString()); + Assert.Equal("keep1\nkeep2", result); } [Fact] - public void FilterLines_CrLfSplitAcrossChunks() + public void FilterLines_CrLfInLongContent() { - // Create scenario where \r is at end of one chunk and \n at start of next var lineContent = new string('y', 7999); - var builder = new StringBuilder(); - builder.Append(lineContent); - builder.Append("\r\n"); - builder.Append("nextline"); - builder.FilterLines(line => line == lineContent); + var result = Filter($"{lineContent}\r\nnextline", _ => _ == lineContent); - Assert.Equal("nextline", builder.ToString()); + Assert.Equal("nextline", result); } [Fact] - public void FilterLines_MultipleLinesSpanningChunks() + public void FilterLines_ManyLongLines() { var builder = new StringBuilder(); var lines = new List(); - // Create lines of varying lengths that will span chunk boundaries for (var i = 0; i < 20; i++) { var line = $"line{i}_" + new string((char) ('a' + i % 26), 1000 + i * 100); @@ -401,28 +242,24 @@ public void FilterLines_MultipleLinesSpanningChunks() } // Remove even-indexed lines - builder.FilterLines(line => lines.IndexOf(line) % 2 == 0); + var result = Filter(builder.ToString(), _ => lines.IndexOf(_) % 2 == 0); - var expected = string.Join('\n', lines.Where((_, i) => i % 2 == 1)); - Assert.Equal(expected, builder.ToString()); + var expected = string.Join('\n', lines.Where((_, index) => index % 2 == 1)); + Assert.Equal(expected, result); } [Fact] public void FilterLines_VeryLongLineKept() { var longLine = new string('z', 20_000); - var builder = new StringBuilder(); - builder.Append("remove\n"); - builder.Append(longLine); - builder.Append("\nremove"); - builder.FilterLines(line => line == "remove"); + var result = Filter($"remove\n{longLine}\nremove", _ => _ == "remove"); - Assert.Equal(longLine, builder.ToString()); + Assert.Equal(longLine, result); } [Fact] - public void FilterLines_MultipleChunksAllRemoved() + public void FilterLines_ManyLinesAllRemoved() { var builder = new StringBuilder(); for (var i = 0; i < 10; i++) @@ -431,58 +268,43 @@ public void FilterLines_MultipleChunksAllRemoved() builder.Append('\n'); } - builder.FilterLines(_ => true); - - Assert.Equal(string.Empty, builder.ToString()); + Assert.Equal(string.Empty, Filter(builder.ToString(), _ => true)); } [Fact] - public void FilterLines_MultipleChunksNoneRemoved() + public void FilterLines_ManyLinesNoneRemoved() { var builder = new StringBuilder(); for (var i = 0; i < 10; i++) { - var line = new string((char) ('a' + i), 2000); - builder.Append(line); + builder.Append((char) ('a' + i), 2000); builder.Append('\n'); } var original = builder.ToString(); - builder.FilterLines(_ => false); - - Assert.Equal(original, builder.ToString()); + Assert.Equal(original, Filter(original, _ => false)); } [Fact] - public void FilterLines_ChunkBoundaryInMiddleOfLine() + public void FilterLines_LongLinePreserved() { - // Specifically target chunk boundary within line content var prefix = new string('p', 7990); var suffix = new string('s', 100); var fullLine = prefix + suffix; - var builder = new StringBuilder(); - builder.Append("before\n"); - builder.Append(fullLine); - builder.Append("\nafter"); - - builder.FilterLines(line => line == "before"); + var result = Filter($"before\n{fullLine}\nafter", _ => _ == "before"); - Assert.Equal(fullLine + "\nafter", builder.ToString()); + Assert.Equal(fullLine + "\nafter", result); } [Fact] - public void FilterLines_EmptyLinesAroundChunkBoundaries() + public void FilterLines_EmptyLinesAfterLongLine() { var longLine = new string('x', 8000); - var builder = new StringBuilder(); - builder.Append(longLine); - builder.Append("\n\n\n"); - builder.Append("keep"); - builder.FilterLines(string.IsNullOrEmpty); + var result = Filter($"{longLine}\n\n\nkeep", string.IsNullOrEmpty); - Assert.Equal(longLine + "\nkeep", builder.ToString()); + Assert.Equal(longLine + "\nkeep", result); } -} \ No newline at end of file +} diff --git a/src/Verify.Tests/UserMachineScrubberChunkTests.cs b/src/Verify.Tests/UserMachineScrubberChunkTests.cs deleted file mode 100644 index 05ae116999..0000000000 --- a/src/Verify.Tests/UserMachineScrubberChunkTests.cs +++ /dev/null @@ -1,38 +0,0 @@ -public class UserMachineScrubberChunkTests -{ - [Fact] - public void CrossChunkMatchEndingExactlyAtChunkBoundary() - { - // "ABCD" fills the capacity-4 chunk; the next append lands in a fresh - // 16-char chunk holding the remaining 16 chars of the 20-char match, so - // the match ends exactly at that chunk's boundary with a chunk after it. - // The trailing-char check must not read chunkSpan[chunkSpan.Length]. - var builder = new StringBuilder(capacity: 4); - builder.Append("ABCD"); - builder.Append("EFGHIJKLMNOPQRST"); - builder.Append('.'); - - UserMachineScrubber.PerformReplacements(builder, "ABCDEFGHIJKLMNOPQRST", "TheUserName"); - - Assert.Equal("TheUserName.", builder.ToString()); - } - - [Fact] - public void TokenSpanningThreeChunks() - { - // Capacity 4 forces small chunks [4][4][…]. The 10-char match spans all - // three, and the 4-char middle chunk is shorter than the match. The - // carryover must accumulate across chunks; otherwise the first chunk's - // prefix is dropped when the short middle chunk overwrites it, and the - // match is never found (silent leak). - var find = "ABCDEFGHIJ"; // 10 chars - var builder = new StringBuilder(capacity: 4); - builder.Append("ABCD"); // chunk0 = find[0..4] - builder.Append("EFGH"); // chunk1 = find[4..8] (short middle chunk) - builder.Append("IJ."); // chunk2 = find[8..10] + wrapper - - UserMachineScrubber.PerformReplacements(builder, find, "TheUserName"); - - Assert.Equal("TheUserName.", builder.ToString()); - } -} diff --git a/src/Verify.slnx b/src/Verify.slnx index 3e903a52ea..1dd945c0f0 100644 --- a/src/Verify.slnx +++ b/src/Verify.slnx @@ -13,6 +13,7 @@ + diff --git a/src/Verify/Extensions.cs b/src/Verify/Extensions.cs index 359de280fa..32c26494d5 100644 --- a/src/Verify/Extensions.cs +++ b/src/Verify/Extensions.cs @@ -149,63 +149,6 @@ public static string NameWithParent(this Type type) return attribute?.Configuration; } - public static char? FirstChar(this StringBuilder builder) - { - if (builder.Length > 0) - { - return builder[0]; - } - - return null; - } - - public static char? LastChar(this StringBuilder builder) - { - if (builder.Length > 0) - { - return builder[^1]; - } - - return null; - } - - public static void FilterLines(this StringBuilder input, Func removeLine) - { - var theString = input.ToString(); - using var reader = new StringReader(theString); - input.Clear(); - - while (reader.ReadLine() is { } line) - { - if (removeLine(line)) - { - continue; - } - - input.AppendLineN(line); - } - - var endsWithNewLine = theString.EndsWith('\n'); - if (input.Length > 0 && !endsWithNewLine) - { - input.Length -= 1; - } - } - - public static void RemoveEmptyLines(this StringBuilder builder) - { - builder.FilterLines(string.IsNullOrWhiteSpace); - if (builder.FirstChar() is '\n') - { - builder.Remove(0, 1); - } - - if (builder.LastChar() is '\n') - { - builder.Length--; - } - } - public static string Remove(this string value, string toRemove) => value.Replace(toRemove, ""); diff --git a/src/Verify/Serialization/CustomContractResolver_Dictionary.cs b/src/Verify/Serialization/CustomContractResolver_Dictionary.cs index cde18436ae..06f58eadac 100644 --- a/src/Verify/Serialization/CustomContractResolver_Dictionary.cs +++ b/src/Verify/Serialization/CustomContractResolver_Dictionary.cs @@ -100,7 +100,7 @@ static bool TryConvertDictionaryKey(VerifyJsonWriter writer, object original, [N return true; } - result = ApplyScrubbers.ApplyForPropertyValue(stringValue.AsSpan(), writer.settings, counter); + result = ApplyScrubbers.ApplyForPropertyValue(stringValue, writer.settings, counter); return true; } diff --git a/src/Verify/Serialization/Scrubbers/ApplyScrubbers.cs b/src/Verify/Serialization/Scrubbers/ApplyScrubbers.cs index 4dba9a4e0f..5f94469b41 100644 --- a/src/Verify/Serialization/Scrubbers/ApplyScrubbers.cs +++ b/src/Verify/Serialization/Scrubbers/ApplyScrubbers.cs @@ -1,5 +1,3 @@ -// ReSharper disable RedundantSuppressNullableWarningExpression - static class ApplyScrubbers { public static void ApplyForExtension(string extension, StringBuilder target, VerifySettings settings, Counter counter) @@ -10,6 +8,25 @@ public static void ApplyForExtension(string extension, StringBuilder target, Ver return; } + var set = EngineScrubberSet.ForExtension(settings, extension); + var source = target.ToString(); + + if (!HasLegacyForExtension(settings, extension)) + { + ScrubEngine.RunToBuilder(source, set, counter, settings.Context, applyDirectoryReplacements: true, target); + return; + } + + // Span scrubbers run first, then the legacy pass over the intermediate result. + // Path replacements and newline normalization stay after the legacy pass so + // legacy scrubbers keep seeing raw paths and may inject '\r'. + var intermediate = ScrubEngine.Run(source, set, counter, settings.Context, applyDirectoryReplacements: false); + if (!ReferenceEquals(intermediate, source)) + { + target.Clear(); + target.Append(intermediate); + } + if (settings.InstanceScrubbers != null) { foreach (var scrubber in settings.InstanceScrubbers) @@ -45,40 +62,67 @@ public static void ApplyForExtension(string extension, StringBuilder target, Ver target.FixNewlines(); } - public static string ApplyForPropertyValue(CharSpan value, VerifySettings settings, Counter counter) + // Fast path: when nothing can change the value, skip all scrubbing work + // (this is the hottest loop in the library — one call per serialized string + // value). Returns the input span unchanged (zero alloc) when possible. + public static CharSpan ApplyForPropertyValue(CharSpan value, VerifySettings settings, Counter counter) { - // Fast path: when nothing can change the value, skip the StringBuilder - // round-trip and the directory-replacement scan (this is the hottest loop - // in the library — one call per serialized string value). - if (!value.Contains('\r')) + if (!settings.ScrubbersEnabled) { - if (!settings.ScrubbersEnabled) + if (!value.Contains('\r')) { - return value.ToString(); + return value; } - if (settings.InstanceScrubbers == null && - VerifierSettings.GlobalScrubbers.Count == 0 && - value.Length < DirectoryReplacements.ShortestFindLength) + return Scrubber.NormalizeNewlines(value.ToString()).AsSpan(); + } + + if (!HasLegacyForPropertyValue(settings)) + { + var set = EngineScrubberSet.ForPropertyValue(settings); + var minTrigger = Math.Min(set.MinTrigger, DirectoryReplacements.ShortestFindLength); + if (value.Length < minTrigger && + !value.Contains('\r')) { - return value.ToString(); + return value; } + + return ScrubEngine.Run(value.ToString(), set, counter, settings.Context, applyDirectoryReplacements: true).AsSpan(); } - var builder = new StringBuilder(value.Length); - builder.Append(value); - ApplyForPropertyValue(settings, counter, builder); - return builder.ToString(); + return ApplyWithLegacy(value.ToString(), settings, counter).AsSpan(); } - public static void ApplyForPropertyValue(VerifySettings settings, Counter counter, StringBuilder builder) + // String variant: returns the same string instance when nothing changed + public static string ApplyForPropertyValue(string value, VerifySettings settings, Counter counter) { if (!settings.ScrubbersEnabled) { - builder.FixNewlines(); - return; + return Scrubber.NormalizeNewlines(value); } + if (!HasLegacyForPropertyValue(settings)) + { + var set = EngineScrubberSet.ForPropertyValue(settings); + var minTrigger = Math.Min(set.MinTrigger, DirectoryReplacements.ShortestFindLength); + if (value.Length < minTrigger && + !value.Contains('\r')) + { + return value; + } + + return ScrubEngine.Run(value, set, counter, settings.Context, applyDirectoryReplacements: true); + } + + return ApplyWithLegacy(value, settings, counter); + } + + static string ApplyWithLegacy(string value, VerifySettings settings, Counter counter) + { + var set = EngineScrubberSet.ForPropertyValue(settings); + var intermediate = ScrubEngine.Run(value, set, counter, settings.Context, applyDirectoryReplacements: false); + + var builder = new StringBuilder(intermediate); if (settings.InstanceScrubbers != null) { foreach (var scrubber in settings.InstanceScrubbers) @@ -95,8 +139,19 @@ public static void ApplyForPropertyValue(VerifySettings settings, Counter counte DirectoryReplacements.Replace(builder); builder.FixNewlines(); + return builder.ToString(); } - static string CleanPath(string directory) => - directory.TrimEnd('/', '\\'); -} \ No newline at end of file + static bool HasLegacyForExtension(VerifySettings settings, string extension) => + settings.InstanceScrubbers is { Count: > 0 } || + (settings.ExtensionMappedInstanceScrubbers != null && + settings.ExtensionMappedInstanceScrubbers.TryGetValue(extension, out var extensionInstance) && + extensionInstance.Count > 0) || + (VerifierSettings.ExtensionMappedGlobalScrubbers.TryGetValue(extension, out var extensionGlobal) && + extensionGlobal.Count > 0) || + VerifierSettings.GlobalScrubbers.Count > 0; + + static bool HasLegacyForPropertyValue(VerifySettings settings) => + settings.InstanceScrubbers is { Count: > 0 } || + VerifierSettings.GlobalScrubbers.Count > 0; +} diff --git a/src/Verify/Serialization/Scrubbers/DateMatchers.cs b/src/Verify/Serialization/Scrubbers/DateMatchers.cs new file mode 100644 index 0000000000..616a8deefe --- /dev/null +++ b/src/Verify/Serialization/Scrubbers/DateMatchers.cs @@ -0,0 +1,204 @@ +// The inline date scrubbers: window scrubbers whose bounds come from +// DateFormatLengthCalculator and whose matcher is a TryParseExact for the format. +// Formats ending in upper case fraction specifiers (.F to .FFFF) produce a second +// scrubber for the trimmed format, since those fractions render as empty when zero; +// the longer max of the untrimmed scrubber naturally orders it first. +// The culture (CurrentCulture when null) is resolved when the scrubber is created. +static class DateMatchers +{ + // A probe date within the supported range of every calendar + // (DateTime.MaxValue is out of range for e.g. the UmAlQura calendar) + static DateTime probeDateTime = new(2000, 1, 1, 1, 1, 1, DateTimeKind.Utc); + + public static Scrubber[] DateTimes( + [StringSyntax(StringSyntaxAttribute.DateTimeFormat)] string format, + Culture? culture) + { + var resolvedCulture = culture ?? Culture.CurrentCulture; + try + { + probeDateTime.ToString(format, resolvedCulture); + } + catch (FormatException exception) + { + throw new($"Format '{format}' is not valid for DateTime.ToString(format, culture).", exception); + } + + return BuildForFormats( + format, + resolvedCulture, + static (format, culture) => Single( + format, + culture, + (window, counter) => + { + if (DateTime.TryParseExact(window, format, culture, DateTimeStyles.None, out var date)) + { + return counter.Convert(date); + } + + return null; + })); + } + + public static Scrubber[] DateTimeOffsets( + [StringSyntax(StringSyntaxAttribute.DateTimeFormat)] string format, + Culture? culture) + { + var resolvedCulture = culture ?? Culture.CurrentCulture; + try + { + new DateTimeOffset(probeDateTime).ToString(format, resolvedCulture); + } + catch (FormatException exception) + { + throw new($"Format '{format}' is not valid for DateTimeOffset.ToString(format, culture).", exception); + } + + return BuildForFormats( + format, + resolvedCulture, + static (format, culture) => Single( + format, + culture, + (window, counter) => + { + if (DateTimeOffset.TryParseExact(window, format, culture, DateTimeStyles.None, out var date)) + { + return counter.Convert(date); + } + + return null; + })); + } + +#if NET6_0_OR_GREATER + + public static Scrubber[] Dates( + [StringSyntax(StringSyntaxAttribute.DateOnlyFormat)] string format, + Culture? culture) + { + var resolvedCulture = culture ?? Culture.CurrentCulture; + try + { + Date.FromDateTime(probeDateTime).ToString(format, resolvedCulture); + } + catch (FormatException exception) + { + throw new($"Format '{format}' is not valid for DateOnly.ToString(format, culture).", exception); + } + + return BuildForFormats( + format, + resolvedCulture, + static (format, culture) => Single( + format, + culture, + (window, counter) => + { + if (Date.TryParseExact(window, format, culture, DateTimeStyles.None, out var date)) + { + return counter.Convert(date); + } + + return null; + })); + } + +#endif + + delegate string? ParseWindow(CharSpan window, Counter counter); + + static Scrubber[] BuildForFormats(string format, Culture culture, Func build) + { + if (TryGetFormatWithUpperMillisecondsTrimmed(format, out var trimmedFormat)) + { + return + [ + build(format, culture), + build(trimmedFormat, culture) + ]; + } + + return [build(format, culture)]; + } + + static Scrubber Single(string format, Culture culture, ParseWindow parse) + { + var (max, min) = DateFormatLengthCalculator.GetLength(format, culture); + var digitPrefilter = HasDigitPrefilter(format); + return Scrubber.Window( + Math.Max(1, min), + max, + (window, counter, _) => + { + if (!counter.ScrubDateTimes) + { + return null; + } + + if (digitPrefilter && + !char.IsDigit(window[0])) + { + return null; + } + + return parse(window, counter); + }); + } + + // True when the format is guaranteed to render a digit first, allowing the + // parse to be skipped cheaply. Single char (standard) formats expand per + // culture, so they are not prefiltered. + static bool HasDigitPrefilter(string format) + { + if (format.Length < 2) + { + return false; + } + + var first = format[0]; + switch (first) + { + case 'y' or 'H' or 'h' or 'm' or 's' or 'f' or 'F': + return true; + case 'd' or 'M': + // 1 or 2 repeats render digits; 3+ render names + return format.Length < 3 || + format[1] != first || + format[2] != first; + default: + return false; + } + } + + static bool TryGetFormatWithUpperMillisecondsTrimmed(string format, [NotNullWhen(true)] out string? trimmedFormat) + { + if (format.EndsWith(".FFFF")) + { + trimmedFormat = format[..^5]; + return true; + } + + if (format.EndsWith(".FFF")) + { + trimmedFormat = format[..^4]; + return true; + } + + if (format.EndsWith(".FF")) + { + trimmedFormat = format[..^3]; + return true; + } + + if (format.EndsWith(".F")) + { + trimmedFormat = format[..^2]; + return true; + } + + trimmedFormat = null; + return false; + } +} diff --git a/src/Verify/Serialization/Scrubbers/DateScrubber.cs b/src/Verify/Serialization/Scrubbers/DateScrubber.cs deleted file mode 100644 index bfdafc9b7c..0000000000 --- a/src/Verify/Serialization/Scrubbers/DateScrubber.cs +++ /dev/null @@ -1,274 +0,0 @@ -// ReSharper disable ReturnValueOfPureMethodIsNotUsed -static class DateScrubber -{ - delegate bool TryConvert( - CharSpan span, - string format, - Counter counter, - Culture culture, - [NotNullWhen(true)] out string? result); - -#if NET6_0_OR_GREATER - - static bool TryConvertDate( - CharSpan span, - [StringSyntax(StringSyntaxAttribute.DateOnlyFormat)] string format, - Counter counter, - Culture culture, - [NotNullWhen(true)] out string? result) - { - if (Date.TryParseExact(span, format, culture, DateTimeStyles.None, out var date)) - { - result = counter.Convert(date); - return true; - } - - result = null; - return false; - } - - public static Action BuildDateScrubber( - [StringSyntax(StringSyntaxAttribute.DateOnlyFormat)] string format, - Culture? culture) - { - try - { - Date.MaxValue.ToString(format, culture); - } - catch (FormatException exception) - { - throw new($"Format '{format}' is not valid for DateOnly.ToString(format, culture).", exception); - } - - return (builder, counter) => ReplaceDates(builder, format, counter, culture ?? Culture.CurrentCulture); - } - - public static void ReplaceDates( - StringBuilder builder, - [StringSyntax(StringSyntaxAttribute.DateOnlyFormat)] string format, - Counter counter, - Culture culture) => - ReplaceInner( - builder, - format, - counter, - culture, - TryConvertDate); -#endif - - public static Action BuildDateTimeOffsetScrubber( - [StringSyntax(StringSyntaxAttribute.DateTimeFormat)] string format, - Culture? culture) - { - try - { - DateTimeOffset.MaxValue.ToString(format, culture); - } - catch (FormatException exception) - { - throw new($"Format '{format}' is not valid for DateTimeOffset.ToString(format, culture).", exception); - } - - return (builder, counter) => ReplaceDateTimeOffsets(builder, format, counter, culture ?? Culture.CurrentCulture); - } - - static bool TryConvertDateTimeOffset( - CharSpan span, - [StringSyntax(StringSyntaxAttribute.DateTimeFormat)] string format, - Counter counter, - Culture culture, - [NotNullWhen(true)] out string? result) - { - if (DateTimeOffset.TryParseExact(span, format, culture, DateTimeStyles.None, out var date)) - { - result = counter.Convert(date); - return true; - } - - result = null; - return false; - } - - public static void ReplaceDateTimeOffsets( - StringBuilder builder, - [StringSyntax(StringSyntaxAttribute.DateTimeFormat)] string format, - Counter counter, - Culture culture) - { - ReplaceInner( - builder, - format, - counter, - culture, - TryConvertDateTimeOffset); - if (TryGetFormatWithUpperMillisecondsTrimmed(format, out var trimmedFormat)) - { - ReplaceInner( - builder, - trimmedFormat, - counter, - culture, - TryConvertDateTimeOffset); - } - } - - static bool TryConvertDateTime( - CharSpan span, - [StringSyntax(StringSyntaxAttribute.DateTimeFormat)] string format, - Counter counter, - Culture culture, - [NotNullWhen(true)] out string? result) - { - if (DateTime.TryParseExact(span, format, culture, DateTimeStyles.None, out var date)) - { - result = counter.Convert(date); - return true; - } - - result = null; - return false; - } - - public static Action BuildDateTimeScrubber(string format, Culture? culture) - { - try - { - DateTime.MaxValue.ToString(format, culture); - } - catch (FormatException exception) - { - throw new($"Format '{format}' is not valid for DateTime.ToString(format, culture).", exception); - } - - return (builder, counter) => ReplaceDateTimes(builder, format, counter, culture ?? Culture.CurrentCulture); - } - - public static void ReplaceDateTimes(StringBuilder builder, string format, Counter counter, Culture culture) - { - ReplaceInner( - builder, - format, - counter, - culture, - TryConvertDateTime); - if (TryGetFormatWithUpperMillisecondsTrimmed(format, out var trimmedFormat)) - { - ReplaceInner( - builder, - trimmedFormat, - counter, - culture, - TryConvertDateTime); - } - } - - static bool TryGetFormatWithUpperMillisecondsTrimmed(string format, [NotNullWhen(true)] out string? trimmedFormat) - { - if (format.EndsWith(".FFFF")) - { - trimmedFormat = format[..^5]; - return true; - } - - if (format.EndsWith(".FFF")) - { - trimmedFormat = format[..^4]; - return true; - } - - if (format.EndsWith(".FF")) - { - trimmedFormat = format[..^3]; - return true; - } - - if (format.EndsWith(".F")) - { - trimmedFormat = format[..^2]; - return true; - } - - trimmedFormat = null; - return false; - } - - static void ReplaceInner(StringBuilder builder, string format, Counter counter, Culture culture, TryConvert tryConvertDate) - { - if (!counter.ScrubDateTimes) - { - return; - } - - var (max, min) = DateFormatLengthCalculator.GetLength(format, culture); - - if (builder.Length < min) - { - return; - } - - if (min == max) - { - ReplaceFixedLength(builder, format, counter, culture, tryConvertDate, max); - - return; - } - - ReplaceVariableLength(builder, format, counter, culture, tryConvertDate, max, min); - } - - static void ReplaceVariableLength(StringBuilder builder, string format, Counter counter, Culture culture, TryConvert tryConvertDate, int longest, int shortest) - { - var value = builder.AsSpan(); - var builderIndex = 0; - for (var index = 0; index <= value.Length; index++) - { - var found = false; - for (var length = longest; length >= shortest; length--) - { - var end = index + length; - if (end > value.Length) - { - continue; - } - - var slice = value.Slice(index, length); - if (tryConvertDate(slice, format, counter, culture, out var convert)) - { - builder.Overwrite(convert, builderIndex, length); - builderIndex += convert.Length; - index += length - 1; - found = true; - break; - } - } - - if (found) - { - continue; - } - - builderIndex++; - } - } - - static void ReplaceFixedLength(StringBuilder builder, string format, Counter counter, Culture culture, TryConvert tryConvertDate, int length) - { - var value = builder.AsSpan(); - var builderIndex = 0; - var increment = length - 1; - for (var index = 0; index <= value.Length - length; index++) - { - var slice = value.Slice(index, length); - if (tryConvertDate(slice, format, counter, culture, out var convert)) - { - builder.Overwrite(convert, builderIndex, length); - builderIndex += convert.Length; - index += increment; - } - else - { - builderIndex++; - } - } - } -} \ No newline at end of file diff --git a/src/Verify/Serialization/Scrubbers/DirectoryReplacements.cs b/src/Verify/Serialization/Scrubbers/DirectoryReplacements.cs index ca22140ae0..8dafa9f204 100644 --- a/src/Verify/Serialization/Scrubbers/DirectoryReplacements.cs +++ b/src/Verify/Serialization/Scrubbers/DirectoryReplacements.cs @@ -2,6 +2,24 @@ static partial class DirectoryReplacements { + public readonly struct Pair + { + public Pair(string find, string replace) + { +#if DEBUG + if (find.Contains('\\')) + { + throw new("Slashes should be sanitized"); + } +#endif + Find = find; + Replace = replace; + } + + public string Find { get; } + public string Replace { get; } + } + static List items = []; static int shortestFindLength = int.MaxValue; @@ -9,9 +27,67 @@ static partial class DirectoryReplacements // too short to contain any replacement. int.MaxValue when there are none. public static int ShortestFindLength => shortestFindLength; + internal static List Items => items; + public static void Replace(StringBuilder builder) => Replace(builder, items); + // Legacy pass entry point: materialize once, scan with the span matcher, apply + // matches position-descending so earlier indexes stay valid. + public static void Replace(StringBuilder builder, List pairs) + { +#if DEBUG + var finds = pairs.Select(_ => _.Find).ToList(); + if (!finds.OrderByDescending(_ => _.Length).SequenceEqual(finds)) + { + throw new("Pairs should be ordered"); + } + + if (finds.Count != finds.Distinct().Count()) + { + throw new("Find should be distinct"); + } +#endif + if (pairs.Count == 0 || + builder.Length == 0) + { + return; + } + + // pairs are ordered by length desc, so the last is the shortest. If the + // builder is shorter than that, no pair can match. + var shortest = pairs[^1].Find.Length; + if (builder.Length < shortest) + { + return; + } + + var span = builder.ToString().AsSpan(); + List<(int Index, int Length, string Value)>? matches = null; + for (var position = 0; position + shortest <= span.Length; position++) + { + if (!TryMatchAt(span, position, null, null, pairs, out var matchLength, out var replacement)) + { + continue; + } + + matches ??= []; + matches.Add((position, matchLength, replacement)); + position += matchLength - 1; + } + + if (matches == null) + { + return; + } + + for (var index = matches.Count - 1; index >= 0; index--) + { + var match = matches[index]; + builder.Overwrite(match.Value, match.Index, match.Length); + } + } + public static void UseAssembly(string? solutionDir, string projectDir) { var values = new List(); diff --git a/src/Verify/Serialization/Scrubbers/DirectoryReplacements_Span.cs b/src/Verify/Serialization/Scrubbers/DirectoryReplacements_Span.cs new file mode 100644 index 0000000000..dcaf94a59c --- /dev/null +++ b/src/Verify/Serialization/Scrubbers/DirectoryReplacements_Span.cs @@ -0,0 +1,105 @@ +// The span based path matcher shared by the engine pass and the legacy StringBuilder pass. +// Matching rules: '/' and '\' are equivalent (Find is pre-sanitized to '/'); the char before a match +// must not be a letter or digit; a trailing letter or digit invalidates the match; a trailing '/' or +// '\' within the current segment is greedily absorbed into the match. Pairs are ordered longest +// first, so at a given position the most specific path wins. +static partial class DirectoryReplacements +{ + // Attempts to match any pair at the given position within the span. + // beforeSegment/afterSegment are the neighbor chars when the position touches the + // segment edge (null at the document edge). Greedy trailing separator absorption + // never crosses the segment boundary. + internal static bool TryMatchAt( + CharSpan span, + int position, + char? beforeSegment, + char? afterSegment, + List pairs, + out int matchLength, + out string replacement) + { + matchLength = 0; + replacement = string.Empty; + + var preceding = position > 0 ? span[position - 1] : beforeSegment; + if (preceding is { } precedingChar && + char.IsLetterOrDigit(precedingChar)) + { + return false; + } + + foreach (var pair in pairs) + { + var find = pair.Find; + var end = position + find.Length; + if (end > span.Length) + { + continue; + } + + if (!IsPathMatch(span.Slice(position, find.Length), find)) + { + continue; + } + + if (end < span.Length) + { + var trailing = span[end]; + if (char.IsLetterOrDigit(trailing)) + { + continue; + } + + matchLength = find.Length; + + // Greedy: include a trailing separator + if (trailing is '/' or '\\') + { + matchLength++; + } + } + else + { + if (afterSegment is { } after && + char.IsLetterOrDigit(after)) + { + continue; + } + + matchLength = find.Length; + } + + replacement = pair.Replace; + return true; + } + + return false; + } + + // Treat '/' and '\' as equivalent. Find never contains '\'. + static bool IsPathMatch(CharSpan candidate, string find) + { + for (var index = 0; index < find.Length; index++) + { + var candidateChar = candidate[index]; + var findChar = find[index]; + + if (candidateChar is '/' or '\\') + { + if (findChar != '/') + { + return false; + } + + continue; + } + + if (candidateChar != findChar) + { + return false; + } + } + + return true; + } +} diff --git a/src/Verify/Serialization/Scrubbers/DirectoryReplacements_StringBuilder.cs b/src/Verify/Serialization/Scrubbers/DirectoryReplacements_StringBuilder.cs deleted file mode 100644 index 653eca376d..0000000000 --- a/src/Verify/Serialization/Scrubbers/DirectoryReplacements_StringBuilder.cs +++ /dev/null @@ -1,354 +0,0 @@ -static partial class DirectoryReplacements -{ - public readonly struct Pair - { - public Pair(string find, string replace) - { -#if DEBUG - if (find.Contains('\\')) - { - throw new("Slashes should be sanitized"); - } -#endif - Find = find; - Replace = replace; - } - - public string Find { get; } - public string Replace { get; } - } - - public static void Replace(StringBuilder builder, List paths) - { -#if DEBUG - var finds = paths.Select(_ => _.Find).ToList(); - if (!finds.OrderByDescending(_ => _.Length).SequenceEqual(finds)) - { - throw new("Pairs should be ordered"); - } - - if (finds.Count != finds.Distinct().Count()) - { - throw new("Find should be distinct"); - } -#endif - if (builder.Length == 0) - { - return; - } - - var matches = FindMatches(builder, paths); - - // Sort by position descending. In-place to avoid LINQ allocation - matches.Sort((a, b) => b.Index.CompareTo(a.Index)); - - // Apply matches - foreach (var match in matches) - { - builder.Overwrite(match.Value, match.Index, match.Length); - } - } - - static List FindMatches(StringBuilder builder, List pairs) - { - if (pairs.Count == 0) - { - return []; - } - - // pairs are ordered by length desc, so the last is the shortest. If the - // builder is shorter than that, no pair can match. - if (builder.Length < pairs[^1].Find.Length) - { - return []; - } - - var matches = new List(); - // Track matched positions - var matchedRanges = new List<(int Start, int End)>(); - var absolutePosition = 0; - - // pairs are ordered by length. so max length is the first one - var maxLength = pairs[0].Find.Length; - var carryoverSize = maxLength - 1; - - Span carryoverBuffer = stackalloc char[carryoverSize]; - Span combinedBuffer = stackalloc char[maxLength * 2]; - var carryoverLength = 0; - var previousChunkAbsoluteEnd = 0; - - foreach (var chunk in builder.GetChunks()) - { - var chunkSpan = chunk.Span; - - // Check for matches spanning from previous chunk to current chunk - if (carryoverLength > 0) - { - for (var carryoverIndex = 0; carryoverIndex < carryoverLength; carryoverIndex++) - { - foreach (var pair in pairs) - { - var remainingInCarryover = carryoverLength - carryoverIndex; - var neededFromCurrent = pair.Find.Length - remainingInCarryover; - - if (neededFromCurrent <= 0 || - neededFromCurrent > chunkSpan.Length) - { - continue; - } - - var combinedLength = remainingInCarryover + neededFromCurrent; - carryoverBuffer.Slice(carryoverIndex, remainingInCarryover).CopyTo(combinedBuffer); - chunkSpan[..neededFromCurrent].CopyTo(combinedBuffer[remainingInCarryover..]); - - var startPosition = previousChunkAbsoluteEnd - carryoverLength + carryoverIndex; - - // Check if this position overlaps with existing match - if (OverlapsExistingMatch(startPosition, pair.Find.Length, matchedRanges)) - { - continue; - } - - if (!TryMatchAtCrossChunk( - builder, - combinedBuffer[..combinedLength], - chunkSpan, - startPosition, - neededFromCurrent, - pair.Find, - out var matchLength)) - { - continue; - } - - matches.Add(new(startPosition, matchLength, pair.Replace)); - matchedRanges.Add((startPosition, startPosition + matchLength)); - // Found a match at this position, skip other pairs - break; - } - } - } - - // Process matches entirely within this chunk - for (var chunkIndex = 0; chunkIndex < chunk.Length; chunkIndex++) - { - var absoluteIndex = absolutePosition + chunkIndex; - - // Skip if already matched - if (IsPositionMatched(absoluteIndex, matchedRanges)) - { - continue; - } - - foreach (var pair in pairs) - { - // Check if we have enough characters left in this chunk - if (chunkIndex + pair.Find.Length > chunk.Length) - { - continue; - } - - // Check if this would overlap with existing match - if (OverlapsExistingMatch(absoluteIndex, pair.Find.Length, matchedRanges)) - { - continue; - } - - // Try to match at this position - if (!TryMatchAt(chunk, chunkIndex, pair.Find, out var matchLength)) - { - continue; - } - - matches.Add(new(absoluteIndex, matchLength, pair.Replace)); - matchedRanges.Add((absoluteIndex, absoluteIndex + matchLength)); - // Skip past this match - chunkIndex += matchLength - 1; - // Found a match, skip other pairs at this position - break; - } - } - - // Roll the carryover forward: keep the last carryoverSize chars of - // everything seen so far. Rebuilding it from the current chunk alone - // drops the prefix when a chunk is shorter than carryoverSize, so a - // path spanning three or more chunks would never be found. - if (chunk.Length >= carryoverSize) - { - chunkSpan.Slice(chunk.Length - carryoverSize, carryoverSize).CopyTo(carryoverBuffer); - carryoverLength = carryoverSize; - } - else - { - var keep = Math.Min(carryoverLength, carryoverSize - chunk.Length); - carryoverBuffer.Slice(carryoverLength - keep, keep).CopyTo(carryoverBuffer); - chunkSpan.CopyTo(carryoverBuffer[keep..]); - carryoverLength = keep + chunk.Length; - } - - previousChunkAbsoluteEnd = absolutePosition + chunk.Length; - absolutePosition += chunk.Length; - } - - return matches; - } - - static bool IsPositionMatched(int position, List<(int Start, int End)> matchedRanges) - { - foreach (var (start, end) in matchedRanges) - { - if (position >= start && position < end) - { - return true; - } - } - - return false; - } - - static bool OverlapsExistingMatch(int start, int length, List<(int Start, int End)> matchedRanges) - { - var end = start + length; - foreach (var range in matchedRanges) - { - // Check if ranges overlap - if (start < range.End && end > range.Start) - { - return true; - } - } - - return false; - } - - static bool TryMatchAtCrossChunk( - StringBuilder builder, - CharSpan combinedSpan, - CharSpan currentChunkSpan, - int absoluteStartPosition, - int neededFromCurrent, - string find, - out int matchLength) - { - matchLength = 0; - - // Check preceding character - if (absoluteStartPosition > 0) - { - var preceding = builder[absoluteStartPosition - 1]; - if (char.IsLetterOrDigit(preceding)) - { - return false; - } - } - - // Check if the path matches - if (!IsPathMatchAt(combinedSpan, 0, find)) - { - return false; - } - - matchLength = find.Length; - - // Check trailing character (it's in the current chunk) - if (neededFromCurrent < currentChunkSpan.Length) - { - var trailing = currentChunkSpan[neededFromCurrent]; - - // Invalid if trailing is letter or digit - if (char.IsLetterOrDigit(trailing)) - { - return false; - } - - // Greedy: include trailing separator - if (trailing is '/' or '\\') - { - matchLength++; - } - } - - return true; - } - - static bool TryMatchAt(ReadOnlyMemory chunk, int chunkPos, string find, out int matchLength) - { - var span = chunk.Span; - matchLength = 0; - - // Check preceding character - if (chunkPos > 0) - { - var preceding = span[chunkPos - 1]; - if (char.IsLetterOrDigit(preceding)) - { - return false; - } - } - - // Check if the path matches - if (!IsPathMatchAt(span, chunkPos, find)) - { - return false; - } - - // Check trailing character - matchLength = find.Length; - var trailingPos = chunkPos + find.Length; - - if (trailingPos >= span.Length) - { - return true; - } - - var trailing = span[trailingPos]; - - // Invalid if trailing is letter or digit - if (char.IsLetterOrDigit(trailing)) - { - return false; - } - - // Greedy: include trailing separator - if (trailing is '/' or '\\') - { - matchLength++; - } - - return true; - } - - static bool IsPathMatchAt(CharSpan chunk, int chunkPos, string find) - { - for (var i = 0; i < find.Length; i++) - { - var chunkChar = chunk[chunkPos + i]; - var findChar = find[i]; - - // Treat / and \ as equivalent - if (chunkChar is '/' or '\\') - { - if (findChar != '/') - { - return false; - } - - continue; - } - - if (chunkChar != findChar) - { - return false; - } - } - - return true; - } - - readonly struct Match(int index, int length, string value) - { - public readonly int Index = index; - public readonly int Length = length; - public readonly string Value = value; - } -} \ No newline at end of file diff --git a/src/Verify/Serialization/Scrubbers/EngineScrubberSet.cs b/src/Verify/Serialization/Scrubbers/EngineScrubberSet.cs new file mode 100644 index 0000000000..ba30ccad4b --- /dev/null +++ b/src/Verify/Serialization/Scrubbers/EngineScrubberSet.cs @@ -0,0 +1,212 @@ +// The merged, ordered view of all registered span scrubbers that apply to a single scrub operation. +// Levels merge in priority order: instance, extension-mapped instance, extension-mapped global, global. +// Inline scrubbers are ordered: unknown max length first, then descending max length, ties broken by +// level then registration order (stable via the collection index). +sealed class EngineScrubberSet +{ + public Scrubber[] LineDrops { get; } + public Scrubber[] LineTransforms { get; } + public Scrubber[] Inline { get; } + + // Values shorter than this cannot be changed by any scrubber in the set. + // 0 disables the fast path. int.MaxValue when the set is empty. + public int MinTrigger { get; } + + // Replicates RemoveEmptyLines trimming one leading and one trailing newline + public bool TrimOuterEmptyLines { get; } + + public bool HasLinePhase => LineDrops.Length > 0 || LineTransforms.Length > 0; + + public bool IsEmpty => + LineDrops.Length == 0 && + LineTransforms.Length == 0 && + Inline.Length == 0; + + static EngineScrubberSet empty = new([], [], []); + + EngineScrubberSet(Scrubber[] lineDrops, Scrubber[] lineTransforms, Scrubber[] inline) + { + LineDrops = lineDrops; + LineTransforms = lineTransforms; + Inline = inline; + + var minTrigger = int.MaxValue; + foreach (var scrubber in lineDrops) + { + if (scrubber.Kind == ScrubberKind.LineDropEmpty) + { + TrimOuterEmptyLines = true; + } + + minTrigger = Math.Min(minTrigger, MinLengthFor(scrubber)); + } + + foreach (var scrubber in lineTransforms) + { + minTrigger = Math.Min(minTrigger, MinLengthFor(scrubber)); + } + + foreach (var scrubber in inline) + { + minTrigger = Math.Min(minTrigger, MinLengthFor(scrubber)); + } + + MinTrigger = minTrigger; + } + + static int MinLengthFor(Scrubber scrubber) => + scrubber.Kind switch + { + // A whitespace-only line needs at least one char to be droppable. + // Removing empty lines from an empty string is the identity. + ScrubberKind.LineDropEmpty => 1, + // Predicates and transforms can act on empty lines + ScrubberKind.LineDropSpan or + ScrubberKind.LineDropString or + ScrubberKind.LineTransformSpan or + ScrubberKind.LineTransformString => 0, + // Replace, Window, LineDropNeedles are at least 1. Match may be 0 (unknown) + _ => scrubber.MinLength + }; + + static ConcurrentDictionary globalExtensionCache = new(StringComparer.Ordinal); + static EngineScrubberSet? globalOnlyCache; + + public static void InvalidateGlobalCache() + { + globalExtensionCache.Clear(); + globalOnlyCache = null; + } + + // Whole-file pass: all four levels + public static EngineScrubberSet ForExtension(VerifySettings settings, string extension) + { + var instance = settings.InstanceSpanScrubbers; + List? extensionInstance = null; + settings.ExtensionMappedInstanceSpanScrubbers?.TryGetValue(extension, out extensionInstance); + + if (IsNullOrEmpty(instance) && + IsNullOrEmpty(extensionInstance)) + { + return globalExtensionCache.GetOrAdd( + extension, + static ext => + { + VerifierSettings.ExtensionMappedGlobalSpanScrubbers.TryGetValue(ext, out var extensionGlobal); + return Build(null, null, extensionGlobal, VerifierSettings.GlobalSpanScrubbers); + }); + } + + VerifierSettings.ExtensionMappedGlobalSpanScrubbers.TryGetValue(extension, out var extensionGlobalScrubbers); + return Build(instance, extensionInstance, extensionGlobalScrubbers, VerifierSettings.GlobalSpanScrubbers); + } + + // Property-value pass: instance and global only. Extension-mapped scrubbers + // are excluded, matching the legacy pipeline. + public static EngineScrubberSet ForPropertyValue(VerifySettings settings) + { + var instance = settings.InstanceSpanScrubbers; + if (IsNullOrEmpty(instance)) + { + return globalOnlyCache ??= Build(null, null, null, VerifierSettings.GlobalSpanScrubbers); + } + + return Build(instance, null, null, VerifierSettings.GlobalSpanScrubbers); + } + + static bool IsNullOrEmpty(List? list) => + list == null || + list.Count == 0; + + // Test hook: a set over explicit scrubbers, bypassing the registration levels + internal static EngineScrubberSet ForScrubbers(List scrubbers) => + Build(scrubbers, null, null, []); + + static EngineScrubberSet Build( + List? instance, + List? extensionInstance, + List? extensionGlobal, + List global) + { + var count = (instance?.Count ?? 0) + + (extensionInstance?.Count ?? 0) + + (extensionGlobal?.Count ?? 0) + + global.Count; + if (count == 0) + { + return empty; + } + + var lineDrops = new List(); + var lineTransforms = new List(); + var inline = new List<(Scrubber Scrubber, int Order)>(); + var order = 0; + + void Collect(List? scrubbers) + { + if (scrubbers == null) + { + return; + } + + foreach (var scrubber in scrubbers) + { + if (scrubber.IsLineDrop) + { + lineDrops.Add(scrubber); + } + else if (scrubber.IsLineTransform) + { + lineTransforms.Add(scrubber); + } + else + { + inline.Add((scrubber, order)); + } + + order++; + } + } + + Collect(instance); + Collect(extensionInstance); + Collect(extensionGlobal); + Collect(global); + + inline.Sort((left, right) => + { + var leftMax = left.Scrubber.MaxLength; + var rightMax = right.Scrubber.MaxLength; + + // Unknown max length runs first + if (leftMax.HasValue != rightMax.HasValue) + { + return leftMax.HasValue ? 1 : -1; + } + + if (leftMax.HasValue) + { + // Longest max first + var compare = rightMax!.Value.CompareTo(leftMax.Value); + if (compare != 0) + { + return compare; + } + } + + // Level then registration order + return left.Order.CompareTo(right.Order); + }); + + var inlineArray = new Scrubber[inline.Count]; + for (var index = 0; index < inline.Count; index++) + { + inlineArray[index] = inline[index].Scrubber; + } + + return new( + [.. lineDrops], + [.. lineTransforms], + inlineArray); + } +} diff --git a/src/Verify/Serialization/Scrubbers/GuidMatcher.cs b/src/Verify/Serialization/Scrubbers/GuidMatcher.cs new file mode 100644 index 0000000000..de8a6efa92 --- /dev/null +++ b/src/Verify/Serialization/Scrubbers/GuidMatcher.cs @@ -0,0 +1,30 @@ +// The inline guid scrubber: a fixed 36 char window over the canonical "D" format, +// with a dash prefilter so the parse is only attempted at plausible positions. +static class GuidMatcher +{ + public static readonly Scrubber Instance = Scrubber.Window(36, 36, Match, requireWordBoundary: true); + + static string? Match(CharSpan window, Counter counter, IReadOnlyDictionary context) + { + if (!counter.ScrubGuids) + { + return null; + } + + // Cheap prefilter: the "D" format has dashes at fixed offsets + if (window[8] != '-' || + window[13] != '-' || + window[18] != '-' || + window[23] != '-') + { + return null; + } + + if (!Guid.TryParseExact(window, "D", out var guid)) + { + return null; + } + + return counter.Convert(guid); + } +} diff --git a/src/Verify/Serialization/Scrubbers/GuidScrubber.cs b/src/Verify/Serialization/Scrubbers/GuidScrubber.cs deleted file mode 100644 index 84162387cb..0000000000 --- a/src/Verify/Serialization/Scrubbers/GuidScrubber.cs +++ /dev/null @@ -1,163 +0,0 @@ -static class GuidScrubber -{ - public static void ReplaceGuids(StringBuilder builder, Counter counter) - { - if (!counter.ScrubGuids) - { - return; - } - - //{173535ae-995b-4cc6-a74e-8cd4be57039c} - if (builder.Length < 36) - { - return; - } - - var matches = FindMatches(builder, counter); - - // Sort by position descending. In-place to avoid LINQ allocation - matches.Sort((a, b) => b.Index.CompareTo(a.Index)); - - // Apply matches - foreach (var match in matches) - { - builder.Overwrite(match.Value, match.Index, 36); - } - } - - static List FindMatches(StringBuilder builder, Counter counter) - { - var absolutePosition = 0; - var matches = new List(); - Span carryoverBuffer = stackalloc char[35]; - Span buffer = stackalloc char[36]; - var carryoverLength = 0; - var previousChunkAbsoluteEnd = 0; - - foreach (var chunk in builder.GetChunks()) - { - var chunkSpan = chunk.Span; - - // Check for GUIDs spanning from previous chunk to current chunk - if (carryoverLength > 0) - { - // Check each possible starting position in the carryover - for (var carryoverIndex = 0; carryoverIndex < carryoverLength; carryoverIndex++) - { - var remainingInCarryover = carryoverLength - carryoverIndex; - var neededFromCurrent = 36 - remainingInCarryover; - - if (neededFromCurrent <= 0 || - chunkSpan.Length < neededFromCurrent) - { - continue; - } - - carryoverBuffer.Slice(carryoverIndex, remainingInCarryover).CopyTo(buffer); - chunkSpan[..neededFromCurrent].CopyTo(buffer[remainingInCarryover..]); - - // Check boundary characters - var startPosition = previousChunkAbsoluteEnd - carryoverLength + carryoverIndex; - - var hasValidStart = startPosition == 0 || - !IsInvalidStartingChar(builder[startPosition - 1]); - - if (!hasValidStart) - { - continue; - } - - var hasValidEnd = neededFromCurrent >= chunkSpan.Length || - !IsInvalidEndingChar(chunkSpan[neededFromCurrent]); - - if (!hasValidEnd) - { - continue; - } - - if (!Guid.TryParseExact(buffer, "D", out var guid)) - { - continue; - } - - var convert = counter.Convert(guid); - matches.Add(new(startPosition, convert)); - } - } - - // Process GUIDs entirely within this chunk - if (chunk.Length >= 36) - { - for (var chunkIndex = 0; chunkIndex < chunk.Length; chunkIndex++) - { - var end = chunkIndex + 36; - if (end > chunk.Length) - { - break; - } - - var value = chunkSpan; - if ((chunkIndex != 0 && IsInvalidStartingChar(value[chunkIndex - 1])) || - (end != value.Length && IsInvalidEndingChar(value[end]))) - { - continue; - } - - var slice = value.Slice(chunkIndex, 36); - - if (!Guid.TryParseExact(slice, "D", out var guid)) - { - continue; - } - - var convert = counter.Convert(guid); - var startReplaceIndex = absolutePosition + chunkIndex; - matches.Add(new(startReplaceIndex, convert)); - chunkIndex += 35; - } - } - - // Roll the carryover forward: keep the last 35 chars of everything seen - // so far. Rebuilding it from the current chunk alone drops the prefix - // when a chunk is shorter than 35, so a guid spanning three or more - // chunks would never be found. - if (chunk.Length >= 35) - { - chunkSpan.Slice(chunk.Length - 35, 35).CopyTo(carryoverBuffer); - carryoverLength = 35; - } - else - { - var keep = Math.Min(carryoverLength, 35 - chunk.Length); - carryoverBuffer.Slice(carryoverLength - keep, keep).CopyTo(carryoverBuffer); - chunkSpan.CopyTo(carryoverBuffer[keep..]); - carryoverLength = keep + chunk.Length; - } - - previousChunkAbsoluteEnd = absolutePosition + chunk.Length; - absolutePosition += chunk.Length; - } - - return matches; - } - - static bool IsInvalidEndingChar(char ch) => - IsInvalidChar(ch) && - ch != '}' && - ch != ')'; - - static bool IsInvalidChar(char ch) => - char.IsLetter(ch) || - char.IsNumber(ch); - - static bool IsInvalidStartingChar(char ch) => - IsInvalidChar(ch) && - ch != '{' && - ch != '('; - - internal readonly struct Match(int index, string value) - { - public readonly int Index = index; - public readonly string Value = value; - } -} \ No newline at end of file diff --git a/src/Verify/Serialization/Scrubbers/LineResult.cs b/src/Verify/Serialization/Scrubbers/LineResult.cs new file mode 100644 index 0000000000..ec6d8d95bf --- /dev/null +++ b/src/Verify/Serialization/Scrubbers/LineResult.cs @@ -0,0 +1,39 @@ +namespace VerifyTests; + +/// +/// The result of processing a line via . +/// +public readonly struct LineResult +{ + internal const byte KeepKind = 0; + internal const byte RemoveKind = 1; + internal const byte ReplaceKind = 2; + + internal byte Kind { get; } + internal string? Text { get; } + + LineResult(byte kind, string? text) + { + Kind = kind; + Text = text; + } + + /// + /// Keep the line unchanged. + /// + public static LineResult Keep { get; } = new(KeepKind, null); + + /// + /// Remove the line. + /// + public static LineResult Remove { get; } = new(RemoveKind, null); + + /// + /// Replace the line with . + /// + public static LineResult Replace(string line) + { + Ensure.NotNull(line); + return new(ReplaceKind, Scrubber.NormalizeNewlines(line)); + } +} diff --git a/src/Verify/Serialization/Scrubbers/LinesScrubber.cs b/src/Verify/Serialization/Scrubbers/LinesScrubber.cs deleted file mode 100644 index 56d480410d..0000000000 --- a/src/Verify/Serialization/Scrubbers/LinesScrubber.cs +++ /dev/null @@ -1,40 +0,0 @@ -static class LinesScrubber -{ - public static void RemoveLinesContaining(this StringBuilder input, StringComparison comparison, params string[] stringToMatch) - { - Ensure.NotNullOrEmpty(stringToMatch); - input.FilterLines(_ => - { - foreach (var toMatch in stringToMatch) - { - if (_.Contains(toMatch, comparison)) - { - return true; - } - } - - return false; - }); - } - - public static void ReplaceLines(this StringBuilder input, Func replaceLine) - { - var theString = input.ToString(); - using var reader = new StringReader(theString); - input.Clear(); - while (reader.ReadLine() is { } line) - { - var value = replaceLine(line); - if (value is not null) - { - input.AppendLineN(value); - } - } - - if (input.Length > 0 && - !theString.EndsWith('\n')) - { - input.Length -= 1; - } - } -} \ No newline at end of file diff --git a/src/Verify/Serialization/Scrubbers/ScrubEngine.cs b/src/Verify/Serialization/Scrubbers/ScrubEngine.cs new file mode 100644 index 0000000000..4b7aaf88cc --- /dev/null +++ b/src/Verify/Serialization/Scrubbers/ScrubEngine.cs @@ -0,0 +1,512 @@ +// The span based scrub engine. Operates on a contiguous source string, tracking the document as a +// list of chunks: scannable source slices and quarantined replacement strings. Each scrubber scans +// the remaining scannable chunks; a match splits its chunk and the replacement is never re-examined +// by other scrubbers. Assembly produces a single string (zero-copy when nothing changed) or fills a +// StringBuilder when a legacy scrubber pass follows. +static partial class ScrubEngine +{ + internal readonly struct Chunk(string text, int start, int length, bool scannable) + { + public readonly string Text = text; + public readonly int Start = start; + public readonly int Length = length; + public readonly bool Scannable = scannable; + + public CharSpan Span => Text.AsSpan(Start, Length); + + public char First => Text[Start]; + + public char Last => Text[Start + Length - 1]; + } + + public static string Run( + string source, + EngineScrubberSet set, + Counter counter, + IReadOnlyDictionary context, + bool applyDirectoryReplacements) + { + var working = Scrubber.NormalizeNewlines(source); + var chunks = ScrubCore(working, set, counter, context, applyDirectoryReplacements); + if (chunks == null) + { + return working; + } + + return AssembleString(chunks); + } + + public static void RunToBuilder( + string source, + EngineScrubberSet set, + Counter counter, + IReadOnlyDictionary context, + bool applyDirectoryReplacements, + StringBuilder target) + { + var working = Scrubber.NormalizeNewlines(source); + var chunks = ScrubCore(working, set, counter, context, applyDirectoryReplacements); + target.Clear(); + if (chunks == null) + { + target.Append(working); + return; + } + + var total = 0; + foreach (var chunk in chunks) + { + total += chunk.Length; + } + + target.EnsureCapacity(total); + foreach (var chunk in chunks) + { + target.Append(chunk.Text, chunk.Start, chunk.Length); + } + } + + // Returns null when nothing changed (the working source is already the result) + static List? ScrubCore( + string source, + EngineScrubberSet set, + Counter counter, + IReadOnlyDictionary context, + bool applyDirectoryReplacements) + { + if (source.Length == 0) + { + return null; + } + + List? chunks = null; + var changed = false; + + if (set.HasLinePhase) + { + chunks = LinePhase(source, set); + changed = chunks != null; + } + + if (set.Inline.Length > 0) + { + chunks ??= [new(source, 0, source.Length, scannable: true)]; + foreach (var scrubber in set.Inline) + { + changed |= ApplyInline(chunks, scrubber, counter, context); + } + } + + // Path replacements are pinned last so user scrubbers always see raw paths + if (applyDirectoryReplacements) + { + var pairs = DirectoryReplacements.Items; + if (pairs.Count > 0 && + source.Length >= DirectoryReplacements.ShortestFindLength) + { + chunks ??= [new(source, 0, source.Length, scannable: true)]; + changed |= ApplyDirectoryReplacements(chunks, pairs); + } + } + + if (changed) + { + return chunks; + } + + return null; + } + + static bool ApplyDirectoryReplacements(List chunks, List pairs) + { + var shortest = pairs[^1].Find.Length; + var changed = false; + var chunkIndex = 0; + while (chunkIndex < chunks.Count) + { + var chunk = chunks[chunkIndex]; + if (!chunk.Scannable || + chunk.Length < shortest) + { + chunkIndex++; + continue; + } + + if (!TryFindDirectoryMatch(chunks, chunkIndex, pairs, shortest, out var matchStart, out var matchLength, out var replacement)) + { + chunkIndex++; + continue; + } + + changed = true; + chunkIndex = Splice(chunks, chunkIndex, matchStart, matchLength, replacement); + } + + return changed; + } + + static bool TryFindDirectoryMatch( + List chunks, + int chunkIndex, + List pairs, + int shortest, + out int matchStart, + out int matchLength, + out string replacement) + { + var span = chunks[chunkIndex].Span; + var beforeSegment = chunkIndex > 0 ? chunks[chunkIndex - 1].Last : (char?) null; + var afterSegment = chunkIndex + 1 < chunks.Count ? chunks[chunkIndex + 1].First : (char?) null; + for (var position = 0; position + shortest <= span.Length; position++) + { + if (DirectoryReplacements.TryMatchAt(span, position, beforeSegment, afterSegment, pairs, out matchLength, out replacement)) + { + matchStart = position; + return true; + } + } + + matchStart = 0; + matchLength = 0; + replacement = string.Empty; + return false; + } + + static string AssembleString(List chunks) + { + var total = 0; + foreach (var chunk in chunks) + { + total += chunk.Length; + } + + if (total == 0) + { + return string.Empty; + } + +#if NET6_0_OR_GREATER + return string.Create( + total, + chunks, + static (span, chunks) => + { + foreach (var chunk in chunks) + { + chunk.Span.CopyTo(span); + span = span[chunk.Length..]; + } + }); +#else + var buffer = new char[total]; + var position = 0; + foreach (var chunk in chunks) + { + chunk.Text.CopyTo(chunk.Start, buffer, position, chunk.Length); + position += chunk.Length; + } + + return new(buffer); +#endif + } + + static bool ApplyInline( + List chunks, + Scrubber scrubber, + Counter counter, + IReadOnlyDictionary context) + { + var changed = false; + var effectiveMin = Math.Max(1, scrubber.MinLength); + var chunkIndex = 0; + while (chunkIndex < chunks.Count) + { + var chunk = chunks[chunkIndex]; + if (!chunk.Scannable || + chunk.Length < effectiveMin) + { + chunkIndex++; + continue; + } + + if (!TryFindMatch(chunks, chunkIndex, scrubber, counter, context, out var matchStart, out var matchLength, out var replacement)) + { + chunkIndex++; + continue; + } + + changed = true; + chunkIndex = Splice(chunks, chunkIndex, matchStart, matchLength, replacement); + } + + return changed; + } + + // Splits the chunk at chunkIndex into [prefix][replacement][suffix]. + // Returns the index of the suffix chunk (to continue scanning), or the index + // after the inserted chunks when the match ended at the chunk end. + static int Splice(List chunks, int chunkIndex, int matchStart, int matchLength, string replacement) + { + var chunk = chunks[chunkIndex]; + chunks.RemoveAt(chunkIndex); + var insertIndex = chunkIndex; + if (matchStart > 0) + { + chunks.Insert(insertIndex, new(chunk.Text, chunk.Start, matchStart, scannable: true)); + insertIndex++; + } + + if (replacement.Length > 0) + { + chunks.Insert(insertIndex, new(replacement, 0, replacement.Length, scannable: false)); + insertIndex++; + } + + var suffixStart = matchStart + matchLength; + var suffixLength = chunk.Length - suffixStart; + if (suffixLength > 0) + { + chunks.Insert(insertIndex, new(chunk.Text, chunk.Start + suffixStart, suffixLength, scannable: true)); + } + + return insertIndex; + } + + static bool TryFindMatch( + List chunks, + int chunkIndex, + Scrubber scrubber, + Counter counter, + IReadOnlyDictionary context, + out int matchStart, + out int matchLength, + out string replacement) + { + switch (scrubber.Kind) + { + case ScrubberKind.Replace: + return TryFindReplaceMatch(chunks, chunkIndex, scrubber, out matchStart, out matchLength, out replacement); + case ScrubberKind.Window: + return TryFindWindowMatch(chunks, chunkIndex, scrubber, counter, context, out matchStart, out matchLength, out replacement); + case ScrubberKind.Match: + return TryFindCustomMatch(chunks, chunkIndex, scrubber, counter, context, out matchStart, out matchLength, out replacement); + default: + throw new($"Unexpected inline scrubber kind: {scrubber.Kind}"); + } + } + + static bool TryFindReplaceMatch( + List chunks, + int chunkIndex, + Scrubber scrubber, + out int matchStart, + out int matchLength, + out string replacement) + { + var span = chunks[chunkIndex].Span; + var pairs = scrubber.Pairs!; + matchStart = -1; + matchLength = 0; + replacement = string.Empty; + + // Pairs are ordered longest first, so at equal positions the longest Find wins + foreach (var (find, pairReplacement) in pairs) + { + if (find.Length > span.Length) + { + continue; + } + + var searchFrom = 0; + while (searchFrom + find.Length <= span.Length) + { + var found = span[searchFrom..].IndexOf(find.AsSpan(), scrubber.Comparison); + if (found < 0) + { + break; + } + + var index = searchFrom + found; + if (matchStart >= 0 && + index >= matchStart) + { + // Cannot beat the current best position + break; + } + + if (scrubber.RequireWordBoundary && + !BoundaryOk(chunks, chunkIndex, index, find.Length)) + { + searchFrom = index + 1; + continue; + } + + matchStart = index; + matchLength = find.Length; + replacement = pairReplacement; + break; + } + } + + return matchStart >= 0; + } + + static bool TryFindWindowMatch( + List chunks, + int chunkIndex, + Scrubber scrubber, + Counter counter, + IReadOnlyDictionary context, + out int matchStart, + out int matchLength, + out string replacement) + { + var span = chunks[chunkIndex].Span; + var min = scrubber.MinLength; + var max = scrubber.MaxLength!.Value; + var matcher = scrubber.WindowMatcher!; + + // Windows never contain a line break, so scan the newline-free regions + var regionStart = 0; + while (regionStart < span.Length) + { + var newlineOffset = span[regionStart..].IndexOf('\n'); + var regionEnd = newlineOffset < 0 ? span.Length : regionStart + newlineOffset; + + for (var position = regionStart; position <= regionEnd - min; position++) + { + if (scrubber.RequireWordBoundary && + PreviousChar(chunks, chunkIndex, position) is { } before && + char.IsLetterOrDigit(before)) + { + continue; + } + + var upper = Math.Min(max, regionEnd - position); + for (var length = upper; length >= min; length--) + { + if (scrubber.RequireWordBoundary && + NextChar(chunks, chunkIndex, position + length) is { } after && + char.IsLetterOrDigit(after)) + { + continue; + } + + var result = matcher(span.Slice(position, length), counter, context); + if (result == null) + { + continue; + } + + matchStart = position; + matchLength = length; + replacement = Scrubber.NormalizeNewlines(result); + return true; + } + } + + if (newlineOffset < 0) + { + break; + } + + regionStart = regionEnd + 1; + } + + matchStart = 0; + matchLength = 0; + replacement = string.Empty; + return false; + } + + static bool TryFindCustomMatch( + List chunks, + int chunkIndex, + Scrubber scrubber, + Counter counter, + IReadOnlyDictionary context, + out int matchStart, + out int matchLength, + out string replacement) + { + var span = chunks[chunkIndex].Span; + if (!scrubber.SegmentMatcher!(span, counter, context, out matchStart, out matchLength, out var result)) + { + replacement = string.Empty; + return false; + } + + if (result == null) + { + throw new("SegmentMatch returned true but no replacement."); + } + + if (matchStart < 0 || + matchLength <= 0 || + matchStart + matchLength > span.Length) + { + throw new($"SegmentMatch returned an invalid range. index: {matchStart}, length: {matchLength}, segment length: {span.Length}."); + } + + if (span.Slice(matchStart, matchLength).Contains('\n')) + { + throw new("SegmentMatch returned a range containing a line break. Matches must not span lines."); + } + + replacement = Scrubber.NormalizeNewlines(result); + return true; + } + + static bool BoundaryOk(List chunks, int chunkIndex, int index, int length) + { + if (PreviousChar(chunks, chunkIndex, index) is { } before && + char.IsLetterOrDigit(before)) + { + return false; + } + + if (NextChar(chunks, chunkIndex, index + length) is { } after && + char.IsLetterOrDigit(after)) + { + return false; + } + + return true; + } + + // The char before the given position within the chunk. At the chunk start the + // neighbor is the last char of the previous chunk (replacement text counts). + static char? PreviousChar(List chunks, int chunkIndex, int position) + { + var chunk = chunks[chunkIndex]; + if (position > 0) + { + return chunk.Text[chunk.Start + position - 1]; + } + + if (chunkIndex > 0) + { + return chunks[chunkIndex - 1].Last; + } + + return null; + } + + // The char at the given position within the chunk. At the chunk end the + // neighbor is the first char of the next chunk (replacement text counts). + static char? NextChar(List chunks, int chunkIndex, int position) + { + var chunk = chunks[chunkIndex]; + if (position < chunk.Length) + { + return chunk.Text[chunk.Start + position]; + } + + if (chunkIndex + 1 < chunks.Count) + { + return chunks[chunkIndex + 1].First; + } + + return null; + } +} diff --git a/src/Verify/Serialization/Scrubbers/ScrubEngine_Lines.cs b/src/Verify/Serialization/Scrubbers/ScrubEngine_Lines.cs new file mode 100644 index 0000000000..d3278d9225 --- /dev/null +++ b/src/Verify/Serialization/Scrubbers/ScrubEngine_Lines.cs @@ -0,0 +1,231 @@ +// The line phase: walks the (already newline-normalized) source once, applying line drops first +// (needle, whitespace, and predicate based) then line transforms in registration order. Drops always +// evaluate the raw line; transform output becomes fresh scannable source for the inline phase. +// Join semantics replicate the legacy StringReader based line scrubbers: lines are joined with \n, +// the trailing newline is preserved only when the original ended with one, and RemoveEmptyLines +// additionally trims the trailing newline. +static partial class ScrubEngine +{ + static string newlineText = "\n"; + + readonly struct LineItem(int start, int end, string? fresh) + { + // Source slice [Start, End) when Fresh is null, otherwise Fresh is replacement line text + public readonly int Start = start; + public readonly int End = end; + public readonly string? Fresh = fresh; + + public int Length => Fresh?.Length ?? End - Start; + } + + // Returns the chunks representing the document after line drops and transforms, + // or null when no line changed. + static List? LinePhase(string source, EngineScrubberSet set) + { + var endsWithNewline = source[^1] == '\n'; + var items = new List(); + var changed = false; + + // Coalesce runs of adjacent untouched kept lines (including their inner + // separators) into single items + int? runStart = null; + var runEnd = 0; + + void FlushRun() + { + if (runStart is { } start) + { + items.Add(new(start, runEnd, null)); + runStart = null; + } + } + + var position = 0; + while (position < source.Length) + { + var newlineIndex = source.IndexOf('\n', position); + var lineEnd = newlineIndex < 0 ? source.Length : newlineIndex; + var lineSpan = source.AsSpan(position, lineEnd - position); + + // Materialized at most once per line, shared by all string based scrubbers + string? lineString = null; + + if (IsDropped(lineSpan, ref lineString, set.LineDrops)) + { + changed = true; + FlushRun(); + } + else + { + var (removed, current) = ApplyTransforms(lineSpan, ref lineString, set.LineTransforms); + if (removed) + { + changed = true; + FlushRun(); + } + else if (current != null && + !lineSpan.SequenceEqual(current.AsSpan())) + { + changed = true; + FlushRun(); + items.Add(new(0, 0, current)); + } + else + { + runStart ??= position; + runEnd = lineEnd; + } + } + + if (newlineIndex < 0) + { + break; + } + + position = newlineIndex + 1; + } + + FlushRun(); + + // RemoveEmptyLines trims the trailing newline even when no line was dropped + if (set.TrimOuterEmptyLines && + endsWithNewline && + items.Count > 0) + { + changed = true; + } + + if (!changed) + { + return null; + } + + var chunks = new List(items.Count * 2); + for (var index = 0; index < items.Count; index++) + { + if (index > 0) + { + chunks.Add(new(newlineText, 0, 1, scannable: true)); + } + + var item = items[index]; + if (item.Length == 0) + { + continue; + } + + if (item.Fresh is { } fresh) + { + // Transformed line text is fresh source, scannable by inline scrubbers + chunks.Add(new(fresh, 0, fresh.Length, scannable: true)); + } + else + { + chunks.Add(new(source, item.Start, item.End - item.Start, scannable: true)); + } + } + + if (endsWithNewline && + items.Count > 0 && + !set.TrimOuterEmptyLines) + { + chunks.Add(new(newlineText, 0, 1, scannable: true)); + } + + return chunks; + } + + static bool IsDropped(CharSpan lineSpan, ref string? lineString, Scrubber[] lineDrops) + { + foreach (var drop in lineDrops) + { + switch (drop.Kind) + { + case ScrubberKind.LineDropNeedles: + if (lineSpan.Length < drop.MinLength) + { + continue; + } + + foreach (var needle in drop.Needles!) + { + if (lineSpan.Contains(needle.AsSpan(), drop.Comparison)) + { + return true; + } + } + + continue; + case ScrubberKind.LineDropEmpty: + if (lineSpan.IsWhiteSpace()) + { + return true; + } + + continue; + case ScrubberKind.LineDropSpan: + if (drop.LineMatcher!(lineSpan)) + { + return true; + } + + continue; + case ScrubberKind.LineDropString: + lineString ??= lineSpan.ToString(); + if (drop.LineStringMatcher!(lineString)) + { + return true; + } + + continue; + default: + throw new($"Unexpected line drop kind: {drop.Kind}"); + } + } + + return false; + } + + static (bool removed, string? current) ApplyTransforms(CharSpan lineSpan, ref string? lineString, Scrubber[] lineTransforms) + { + // The most recent replacement text; null while the line is unchanged + string? current = null; + foreach (var transform in lineTransforms) + { + switch (transform.Kind) + { + case ScrubberKind.LineTransformSpan: + { + var result = transform.LineReplacer!(current is null ? lineSpan : current.AsSpan()); + if (result.Kind == LineResult.RemoveKind) + { + return (true, null); + } + + if (result.Kind == LineResult.ReplaceKind) + { + current = result.Text!; + } + + continue; + } + case ScrubberKind.LineTransformString: + { + var input = current ?? (lineString ??= lineSpan.ToString()); + var output = transform.LineStringReplacer!(input); + if (output == null) + { + return (true, null); + } + + current = Scrubber.NormalizeNewlines(output); + continue; + } + default: + throw new($"Unexpected line transform kind: {transform.Kind}"); + } + } + + return (false, current); + } +} diff --git a/src/Verify/Serialization/Scrubbers/Scrubber.cs b/src/Verify/Serialization/Scrubbers/Scrubber.cs new file mode 100644 index 0000000000..8d19100933 --- /dev/null +++ b/src/Verify/Serialization/Scrubbers/Scrubber.cs @@ -0,0 +1,283 @@ +namespace VerifyTests; + +/// +/// Defines a scrubbing operation executed by the span based scrub engine. +/// +/// +/// +/// Semantics shared by all scrubbers created from this type: +/// +/// +/// Quarantine: text produced by a replacement is never re-examined by other s. +/// Legacy AddScrubber(Action<StringBuilder>) scrubbers run afterwards and can still modify it. +/// Ordering: line removals run first, then line transforms (registration order), then inline +/// scrubbers: unknown max length first, then descending max length, ties broken by registration level +/// (instance, extension mapped instance, extension mapped global, global) then registration order. +/// Path replacements ({ProjectDirectory} etc) always run last. +/// Length rules: text shorter than a scrubber's minimum length is skipped without invoking the scrubber. +/// Single line rule: a match may never contain a line break. Finds may not contain \n or \r. +/// Word boundary: when required, a match is rejected if the character on either side is a letter or digit. +/// +/// +public sealed class Scrubber +{ + internal ScrubberKind Kind { get; } + internal int MinLength { get; } + internal int? MaxLength { get; } + internal StringComparison Comparison { get; } + internal bool RequireWordBoundary { get; } + internal (string Find, string Replacement)[]? Pairs { get; } + internal string[]? Needles { get; } + internal WindowMatch? WindowMatcher { get; } + internal SegmentMatch? SegmentMatcher { get; } + internal LineMatch? LineMatcher { get; } + internal Func? LineStringMatcher { get; } + internal LineReplace? LineReplacer { get; } + internal Func? LineStringReplacer { get; } + + Scrubber( + ScrubberKind kind, + int minLength = 0, + int? maxLength = null, + StringComparison comparison = StringComparison.Ordinal, + bool requireWordBoundary = false, + (string Find, string Replacement)[]? pairs = null, + string[]? needles = null, + WindowMatch? windowMatcher = null, + SegmentMatch? segmentMatcher = null, + LineMatch? lineMatcher = null, + Func? lineStringMatcher = null, + LineReplace? lineReplacer = null, + Func? lineStringReplacer = null) + { + Kind = kind; + MinLength = minLength; + MaxLength = maxLength; + Comparison = comparison; + RequireWordBoundary = requireWordBoundary; + Pairs = pairs; + Needles = needles; + WindowMatcher = windowMatcher; + SegmentMatcher = segmentMatcher; + LineMatcher = lineMatcher; + LineStringMatcher = lineStringMatcher; + LineReplacer = lineReplacer; + LineStringReplacer = lineStringReplacer; + } + + internal bool IsLineDrop => + Kind is ScrubberKind.LineDropNeedles + or ScrubberKind.LineDropSpan + or ScrubberKind.LineDropString + or ScrubberKind.LineDropEmpty; + + internal bool IsLineTransform => + Kind is ScrubberKind.LineTransformSpan + or ScrubberKind.LineTransformString; + + internal bool IsInline => + Kind is ScrubberKind.Replace + or ScrubberKind.Window + or ScrubberKind.Match; + + /// + /// Replace every occurrence of with . + /// + public static Scrubber Replace( + string find, + string replacement, + StringComparison comparison = StringComparison.Ordinal, + bool requireWordBoundary = false) + { + ValidateFind(find); + Ensure.NotNull(replacement); + return new( + ScrubberKind.Replace, + minLength: find.Length, + maxLength: find.Length, + comparison: comparison, + requireWordBoundary: requireWordBoundary, + pairs: [(find, NormalizeNewlines(replacement))]); + } + + /// + /// Replace every occurrence of each Find with its Replacement. + /// At a given position the longest matching Find wins. + /// + public static Scrubber Replace( + StringComparison comparison, + bool requireWordBoundary, + params (string Find, string Replacement)[] pairs) + { + Ensure.NotNullOrEmpty(pairs); + var ordered = new (string Find, string Replacement)[pairs.Length]; + for (var index = 0; index < pairs.Length; index++) + { + var (find, replacement) = pairs[index]; + ValidateFind(find); + Ensure.NotNull(replacement); + ordered[index] = (find, NormalizeNewlines(replacement)); + } + + // Longest first so the most specific Find wins at any given position + Array.Sort(ordered, (left, right) => right.Find.Length.CompareTo(left.Find.Length)); + return new( + ScrubberKind.Replace, + minLength: ordered[^1].Find.Length, + maxLength: ordered[0].Find.Length, + comparison: comparison, + requireWordBoundary: requireWordBoundary, + pairs: ordered); + } + + /// + /// Match candidate windows of text between and characters. + /// At each position the engine tries the longest window first. + /// + public static Scrubber Window( + int minLength, + int maxLength, + WindowMatch matcher, + bool requireWordBoundary = false) + { + Ensure.NotNull(matcher); + if (minLength < 1) + { + throw new ArgumentOutOfRangeException(nameof(minLength), minLength, "minLength must be at least 1."); + } + + if (maxLength < minLength) + { + throw new ArgumentOutOfRangeException(nameof(maxLength), maxLength, "maxLength must be greater than or equal to minLength."); + } + + return new( + ScrubberKind.Window, + minLength: minLength, + maxLength: maxLength, + requireWordBoundary: requireWordBoundary, + windowMatcher: matcher); + } + + /// + /// Find matches using custom search logic. + /// : segments shorter than this are skipped (null scans everything). + /// : used for ordering only; null (unknown) runs before all known length scrubbers. + /// + public static Scrubber Match( + SegmentMatch matcher, + int? minLength = null, + int? maxLength = null) + { + Ensure.NotNull(matcher); + if (minLength is < 1) + { + throw new ArgumentOutOfRangeException(nameof(minLength), minLength, "minLength must be at least 1."); + } + + if (maxLength < minLength) + { + throw new ArgumentOutOfRangeException(nameof(maxLength), maxLength, "maxLength must be greater than or equal to minLength."); + } + + return new( + ScrubberKind.Match, + minLength: minLength ?? 0, + maxLength: maxLength, + segmentMatcher: matcher); + } + + /// + /// Remove any lines containing any of . + /// + public static Scrubber RemoveLinesContaining(StringComparison comparison, params string[] needles) + { + Ensure.NotNullOrEmpty(needles); + var copy = new string[needles.Length]; + var minLength = int.MaxValue; + for (var index = 0; index < needles.Length; index++) + { + var needle = needles[index]; + ValidateFind(needle); + copy[index] = needle; + minLength = Math.Min(minLength, needle.Length); + } + + return new( + ScrubberKind.LineDropNeedles, + minLength: minLength, + comparison: comparison, + needles: copy); + } + + /// + /// Remove any lines containing any of , using . + /// + public static Scrubber RemoveLinesContaining(params string[] needles) => + RemoveLinesContaining(StringComparison.OrdinalIgnoreCase, needles); + + /// + /// Remove any lines matching . + /// + public static Scrubber RemoveLines(LineMatch shouldRemove) + { + Ensure.NotNull(shouldRemove); + return new(ScrubberKind.LineDropSpan, lineMatcher: shouldRemove); + } + + /// + /// Remove any lines matching . + /// + public static Scrubber RemoveLines(Func shouldRemove) + { + Ensure.NotNull(shouldRemove); + return new(ScrubberKind.LineDropString, lineStringMatcher: shouldRemove); + } + + /// + /// Process each line via . + /// + public static Scrubber ReplaceLines(LineReplace replace) + { + Ensure.NotNull(replace); + return new(ScrubberKind.LineTransformSpan, lineReplacer: replace); + } + + /// + /// Process each line via . + /// can return the input to keep the line, a different string to replace it, or null to remove it. + /// + public static Scrubber ReplaceLines(Func replace) + { + Ensure.NotNull(replace); + return new(ScrubberKind.LineTransformString, lineStringReplacer: replace); + } + + /// + /// Remove any lines containing only whitespace. + /// + public static Scrubber RemoveEmptyLines() => + new(ScrubberKind.LineDropEmpty); + + internal static string NormalizeNewlines(string value) + { + if (!value.Contains('\r')) + { + return value; + } + + return value + .Replace("\r\n", "\n") + .Replace('\r', '\n'); + } + + static void ValidateFind(string find) + { + Ensure.NotNullOrEmpty(find); + if (find.Contains('\n') || + find.Contains('\r')) + { + throw new ArgumentException($"Find must not contain line breaks: {find}", nameof(find)); + } + } +} diff --git a/src/Verify/Serialization/Scrubbers/ScrubberKind.cs b/src/Verify/Serialization/Scrubbers/ScrubberKind.cs new file mode 100644 index 0000000000..63923eb2af --- /dev/null +++ b/src/Verify/Serialization/Scrubbers/ScrubberKind.cs @@ -0,0 +1,12 @@ +enum ScrubberKind +{ + Replace, + Window, + Match, + LineDropNeedles, + LineDropSpan, + LineDropString, + LineDropEmpty, + LineTransformSpan, + LineTransformString, +} diff --git a/src/Verify/Serialization/Scrubbers/Scrubber_Delegates.cs b/src/Verify/Serialization/Scrubbers/Scrubber_Delegates.cs new file mode 100644 index 0000000000..4894b88cb9 --- /dev/null +++ b/src/Verify/Serialization/Scrubbers/Scrubber_Delegates.cs @@ -0,0 +1,34 @@ +namespace VerifyTests; + +/// +/// Attempts to match a candidate window of text. +/// The window length is between the minLength and maxLength of the owning and never contains a line break. +/// Return the replacement text, or null when the window is not a match. +/// +public delegate string? WindowMatch( + CharSpan window, + Counter counter, + IReadOnlyDictionary context); + +/// +/// Attempts to find the next match within a segment of text. +/// The segment may contain line breaks, but the matched range must not. +/// Return true and set , , and when a match is found. +/// +public delegate bool SegmentMatch( + CharSpan segment, + Counter counter, + IReadOnlyDictionary context, + out int index, + out int length, + out string? replacement); + +/// +/// Determines if a line matches. The line excludes its terminator. +/// +public delegate bool LineMatch(CharSpan line); + +/// +/// Produces a for a line. The line excludes its terminator. +/// +public delegate LineResult LineReplace(CharSpan line); diff --git a/src/Verify/Serialization/Scrubbers/UserMachineScrubber.cs b/src/Verify/Serialization/Scrubbers/UserMachineScrubber.cs index eaddc697f2..c3a0b0202a 100644 --- a/src/Verify/Serialization/Scrubbers/UserMachineScrubber.cs +++ b/src/Verify/Serialization/Scrubbers/UserMachineScrubber.cs @@ -1,4 +1,4 @@ -static partial class UserMachineScrubber +static class UserMachineScrubber { static string machineName; static string userName; @@ -13,9 +13,9 @@ internal static void ResetReplacements(string machineName, string userName) UserMachineScrubber.userName = userName; } - public static void Machine(StringBuilder builder) => - PerformReplacements(builder, machineName, "TheMachineName"); + public static Scrubber MachineScrubber() => + Scrubber.Replace(machineName, "TheMachineName", StringComparison.Ordinal, requireWordBoundary: true); - public static void User(StringBuilder builder) => - PerformReplacements(builder, userName, "TheUserName"); -} \ No newline at end of file + public static Scrubber UserScrubber() => + Scrubber.Replace(userName, "TheUserName", StringComparison.Ordinal, requireWordBoundary: true); +} diff --git a/src/Verify/Serialization/Scrubbers/UserMachineScrubber_PerformReplacements.cs b/src/Verify/Serialization/Scrubbers/UserMachineScrubber_PerformReplacements.cs deleted file mode 100644 index 0ff8489054..0000000000 --- a/src/Verify/Serialization/Scrubbers/UserMachineScrubber_PerformReplacements.cs +++ /dev/null @@ -1,144 +0,0 @@ -static partial class UserMachineScrubber -{ - static bool IsValidWrapper(char ch) => - !char.IsLetterOrDigit(ch); - - public static void PerformReplacements(StringBuilder builder, string find, string replace) - { - if (builder.Length < find.Length) - { - return; - } - - var matches = FindMatches(builder, find); - - // Sort by position descending. In-place to avoid LINQ allocation - matches.Sort((a, b) => b.CompareTo(a)); - - // Apply matches - foreach (var match in matches) - { - builder.Overwrite(replace, match, find.Length); - } - } - - static List FindMatches(StringBuilder builder, string find) - { - var matches = new List(); - var absolutePosition = 0; - var carryoverSize = find.Length - 1; - - Span carryoverBuffer = stackalloc char[carryoverSize]; - Span combinedBuffer = stackalloc char[find.Length]; - var carryoverLength = 0; - var previousChunkAbsoluteEnd = 0; - - foreach (var chunk in builder.GetChunks()) - { - var chunkSpan = chunk.Span; - - // Check for matches spanning from previous chunk to current chunk - if (carryoverLength > 0) - { - for (var carryoverIndex = 0; carryoverIndex < carryoverLength; carryoverIndex++) - { - var remainingInCarryover = carryoverLength - carryoverIndex; - var neededFromCurrent = find.Length - remainingInCarryover; - - if (neededFromCurrent <= 0 || - neededFromCurrent > chunkSpan.Length) - { - continue; - } - - // Build combined buffer - carryoverBuffer.Slice(carryoverIndex, remainingInCarryover).CopyTo(combinedBuffer); - chunkSpan[..neededFromCurrent].CopyTo(combinedBuffer[remainingInCarryover..]); - - // Check if it matches - if (!combinedBuffer.SequenceEqual(find)) - { - continue; - } - - var startPosition = previousChunkAbsoluteEnd - carryoverLength + carryoverIndex; - - // Check preceding character - var validStart = startPosition == 0 || - IsValidWrapper(builder[startPosition - 1]); - - if (!validStart) - { - continue; - } - - // Check trailing character. Use the builder indexer rather - // than chunkSpan[neededFromCurrent], which is out of range when - // the match ends exactly at this chunk's boundary, so the check - // still works when there is a following chunk. - var endPosition = startPosition + find.Length; - var validEnd = endPosition >= builder.Length || - IsValidWrapper(builder[endPosition]); - - if (!validEnd) - { - continue; - } - - matches.Add(startPosition); - } - } - - // Process matches entirely within this chunk - if (chunk.Length >= find.Length) - { - var chunkIndex = 0; - while (true) - { - var value = chunkSpan; - var searchSpan = value[chunkIndex..]; - var foundIndex = searchSpan.IndexOf(find); - if (foundIndex == -1) - { - break; - } - - chunkIndex += foundIndex; - var end = chunkIndex + find.Length; - - if ((chunkIndex != 0 && !IsValidWrapper(value[chunkIndex - 1])) || - (end != value.Length && !IsValidWrapper(value[end]))) - { - chunkIndex++; - continue; - } - - matches.Add(absolutePosition + chunkIndex); - chunkIndex += find.Length; - } - } - - // Roll the carryover forward: keep the last carryoverSize chars of - // everything seen so far. Rebuilding it from the current chunk alone - // drops the prefix when a chunk is shorter than the search string, so - // a token spanning three or more chunks would never be found. - if (chunk.Length >= carryoverSize) - { - chunkSpan.Slice(chunk.Length - carryoverSize, carryoverSize).CopyTo(carryoverBuffer); - carryoverLength = carryoverSize; - } - else - { - var keep = Math.Min(carryoverLength, carryoverSize - chunk.Length); - carryoverBuffer.Slice(carryoverLength - keep, keep).CopyTo(carryoverBuffer); - chunkSpan.CopyTo(carryoverBuffer[keep..]); - carryoverLength = keep + chunk.Length; - } - - previousChunkAbsoluteEnd = absolutePosition + chunk.Length; - absolutePosition += chunk.Length; - } - - return matches; - } -} \ No newline at end of file diff --git a/src/Verify/Serialization/Scrubbers/VerifierSettings_ExtensionMappedGlobalScrubbers.cs b/src/Verify/Serialization/Scrubbers/VerifierSettings_ExtensionMappedGlobalScrubbers.cs index 3ad5609b3f..ebc019372c 100644 --- a/src/Verify/Serialization/Scrubbers/VerifierSettings_ExtensionMappedGlobalScrubbers.cs +++ b/src/Verify/Serialization/Scrubbers/VerifierSettings_ExtensionMappedGlobalScrubbers.cs @@ -1,9 +1,27 @@ -namespace VerifyTests; +namespace VerifyTests; public static partial class VerifierSettings { internal static Dictionary>>> ExtensionMappedGlobalScrubbers = []; + internal static Dictionary> ExtensionMappedGlobalSpanScrubbers = []; + + /// + /// Add a that applies to all verified files with a matching extension. + /// + public static void AddScrubber(string extension, Scrubber scrubber) + { + InnerVerifier.ThrowIfVerifyHasBeenRun(); + Ensure.NotNull(scrubber); + if (!ExtensionMappedGlobalSpanScrubbers.TryGetValue(extension, out var values)) + { + ExtensionMappedGlobalSpanScrubbers[extension] = values = []; + } + + values.Add(scrubber); + EngineScrubberSet.InvalidateGlobalCache(); + } + /// /// Modify the resulting test content using custom code. /// @@ -45,82 +63,100 @@ public static void AddScrubber(string extension, Action /// Remove any lines containing any of from the test results. /// - public static void ScrubLinesContaining(string extension, StringComparison comparison, params string[] stringToMatch) - { - InnerVerifier.ThrowIfVerifyHasBeenRun(); - ScrubLinesContaining(extension, comparison, ScrubberLocation.First, stringToMatch); - } + public static void ScrubLinesContaining(string extension, StringComparison comparison, params string[] stringToMatch) => + AddScrubber(extension, Scrubber.RemoveLinesContaining(comparison, stringToMatch)); /// /// Remove any lines containing any of from the test results. /// - public static void ScrubLinesContaining(string extension, StringComparison comparison, ScrubberLocation location, params string[] stringToMatch) - { - InnerVerifier.ThrowIfVerifyHasBeenRun(); - AddScrubber(extension, _ => _.RemoveLinesContaining(comparison, stringToMatch), location); - } + [Obsolete(locationObsolete)] + public static void ScrubLinesContaining(string extension, StringComparison comparison, ScrubberLocation location, params string[] stringToMatch) => + ScrubLinesContaining(extension, comparison, stringToMatch); /// /// Remove any lines matching from the test results. /// - public static void ScrubLines(string extension, Func removeLine, ScrubberLocation location = ScrubberLocation.First) - { - InnerVerifier.ThrowIfVerifyHasBeenRun(); - AddScrubber(extension, _ => _.FilterLines(removeLine), location); - } + public static void ScrubLines(string extension, Func removeLine) => + AddScrubber(extension, Scrubber.RemoveLines(removeLine)); + + /// + /// Remove any lines matching from the test results. + /// + [Obsolete(locationObsolete)] + public static void ScrubLines(string extension, Func removeLine, ScrubberLocation location) => + ScrubLines(extension, removeLine); /// /// Remove any lines containing only whitespace from the test results. /// - public static void ScrubEmptyLines(string extension, ScrubberLocation location = ScrubberLocation.First) - { - InnerVerifier.ThrowIfVerifyHasBeenRun(); - AddScrubber(extension, _ => _.RemoveEmptyLines(), location); - } + public static void ScrubEmptyLines(string extension) => + AddScrubber(extension, Scrubber.RemoveEmptyLines()); + + /// + /// Remove any lines containing only whitespace from the test results. + /// + [Obsolete(locationObsolete)] + public static void ScrubEmptyLines(string extension, ScrubberLocation location) => + ScrubEmptyLines(extension); /// /// Replace inline s with a placeholder. /// - public static void ScrubInlineGuids(string extension, ScrubberLocation location = ScrubberLocation.First) - { - InnerVerifier.ThrowIfVerifyHasBeenRun(); - AddScrubber(extension, GuidScrubber.ReplaceGuids, location); - } + public static void ScrubInlineGuids(string extension) => + AddScrubber(extension, GuidMatcher.Instance); + + /// + /// Replace inline s with a placeholder. + /// + [Obsolete(locationObsolete)] + public static void ScrubInlineGuids(string extension, ScrubberLocation location) => + ScrubInlineGuids(extension); /// /// Scrub lines with an optional replace. /// can return the input to ignore the line, or return a different string to replace it. /// - public static void ScrubLinesWithReplace(string extension, Func replaceLine, ScrubberLocation location = ScrubberLocation.First) - { - InnerVerifier.ThrowIfVerifyHasBeenRun(); - AddScrubber(extension, _ => _.ReplaceLines(replaceLine), location); - } + public static void ScrubLinesWithReplace(string extension, Func replaceLine) => + AddScrubber(extension, Scrubber.ReplaceLines(replaceLine)); + + /// + /// Scrub lines with an optional replace. + /// can return the input to ignore the line, or return a different string to replace it. + /// + [Obsolete(locationObsolete)] + public static void ScrubLinesWithReplace(string extension, Func replaceLine, ScrubberLocation location) => + ScrubLinesWithReplace(extension, replaceLine); /// /// Remove any lines containing any of from the test results. /// - public static void ScrubLinesContaining(string extension, ScrubberLocation location = ScrubberLocation.First, params string[] stringToMatch) - { - InnerVerifier.ThrowIfVerifyHasBeenRun(); - ScrubLinesContaining(extension, StringComparison.OrdinalIgnoreCase, location, stringToMatch); - } + [Obsolete(locationObsolete)] + public static void ScrubLinesContaining(string extension, ScrubberLocation location, params string[] stringToMatch) => + ScrubLinesContaining(extension, StringComparison.OrdinalIgnoreCase, stringToMatch); /// /// Remove the from the test results. /// - public static void ScrubMachineName(string extension, ScrubberLocation location = ScrubberLocation.First) - { - InnerVerifier.ThrowIfVerifyHasBeenRun(); - AddScrubber(extension, UserMachineScrubber.Machine, location); - } + public static void ScrubMachineName(string extension) => + AddScrubber(extension, UserMachineScrubber.MachineScrubber()); + + /// + /// Remove the from the test results. + /// + [Obsolete(locationObsolete)] + public static void ScrubMachineName(string extension, ScrubberLocation location) => + ScrubMachineName(extension); /// /// Remove the from the test results. /// - public static void ScrubUserName(string extension, ScrubberLocation location = ScrubberLocation.First) - { - InnerVerifier.ThrowIfVerifyHasBeenRun(); - AddScrubber(extension, UserMachineScrubber.User, location); - } -} \ No newline at end of file + public static void ScrubUserName(string extension) => + AddScrubber(extension, UserMachineScrubber.UserScrubber()); + + /// + /// Remove the from the test results. + /// + [Obsolete(locationObsolete)] + public static void ScrubUserName(string extension, ScrubberLocation location) => + ScrubUserName(extension); +} diff --git a/src/Verify/Serialization/Scrubbers/VerifierSettings_GlobalScrubbers.cs b/src/Verify/Serialization/Scrubbers/VerifierSettings_GlobalScrubbers.cs index cb6732bb40..f4b0ed5c8b 100644 --- a/src/Verify/Serialization/Scrubbers/VerifierSettings_GlobalScrubbers.cs +++ b/src/Verify/Serialization/Scrubbers/VerifierSettings_GlobalScrubbers.cs @@ -1,9 +1,24 @@ -namespace VerifyTests; +namespace VerifyTests; public static partial class VerifierSettings { internal static List>> GlobalScrubbers = []; + internal static List GlobalSpanScrubbers = []; + + const string locationObsolete = "ScrubberLocation is ignored; span scrubber ordering is engine determined. Use the overload without ScrubberLocation."; + + /// + /// Add a that applies to all verified files. + /// + public static void AddScrubber(Scrubber scrubber) + { + InnerVerifier.ThrowIfVerifyHasBeenRun(); + Ensure.NotNull(scrubber); + GlobalSpanScrubbers.Add(scrubber); + EngineScrubberSet.InvalidateGlobalCache(); + } + /// /// Modify the resulting test content using custom code. /// @@ -36,38 +51,41 @@ public static void AddScrubber(Action /// Remove any lines containing any of from the test results. /// - public static void ScrubLinesContaining(StringComparison comparison, params string[] stringToMatch) - { - InnerVerifier.ThrowIfVerifyHasBeenRun(); - ScrubLinesContaining(comparison, ScrubberLocation.First, stringToMatch); - } + public static void ScrubLinesContaining(StringComparison comparison, params string[] stringToMatch) => + AddScrubber(Scrubber.RemoveLinesContaining(comparison, stringToMatch)); /// /// Remove any lines containing any of from the test results. /// - public static void ScrubLinesContaining(StringComparison comparison, ScrubberLocation location, params string[] stringToMatch) - { - InnerVerifier.ThrowIfVerifyHasBeenRun(); - AddScrubber(_ => _.RemoveLinesContaining(comparison, stringToMatch), location); - } + [Obsolete(locationObsolete)] + public static void ScrubLinesContaining(StringComparison comparison, ScrubberLocation location, params string[] stringToMatch) => + ScrubLinesContaining(comparison, stringToMatch); /// /// Remove any lines matching from the test results. /// - public static void ScrubLines(Func removeLine, ScrubberLocation location = ScrubberLocation.First) - { - InnerVerifier.ThrowIfVerifyHasBeenRun(); - AddScrubber(_ => _.FilterLines(removeLine), location); - } + public static void ScrubLines(Func removeLine) => + AddScrubber(Scrubber.RemoveLines(removeLine)); + + /// + /// Remove any lines matching from the test results. + /// + [Obsolete(locationObsolete)] + public static void ScrubLines(Func removeLine, ScrubberLocation location) => + ScrubLines(removeLine); /// /// Remove any lines containing only whitespace from the test results. /// - public static void ScrubEmptyLines(ScrubberLocation location = ScrubberLocation.First) - { - InnerVerifier.ThrowIfVerifyHasBeenRun(); - AddScrubber(_ => _.RemoveEmptyLines(), location); - } + public static void ScrubEmptyLines() => + AddScrubber(Scrubber.RemoveEmptyLines()); + + /// + /// Remove any lines containing only whitespace from the test results. + /// + [Obsolete(locationObsolete)] + public static void ScrubEmptyLines(ScrubberLocation location) => + ScrubEmptyLines(); internal static bool DateCountingEnabled { get; private set; } = true; @@ -82,22 +100,46 @@ public static void DisableDateCounting() => /// public static void ScrubInlineDateTimes( [StringSyntax(StringSyntaxAttribute.DateTimeFormat)] string format, - Culture? culture = null, - ScrubberLocation location = ScrubberLocation.First) => - AddScrubber( - DateScrubber.BuildDateTimeScrubber(format, culture), - location); + Culture? culture = null) + { + foreach (var scrubber in DateMatchers.DateTimes(format, culture)) + { + AddScrubber(scrubber); + } + } /// /// Replace inline s with a placeholder. /// + [Obsolete(locationObsolete)] + public static void ScrubInlineDateTimes( + [StringSyntax(StringSyntaxAttribute.DateTimeFormat)] string format, + Culture? culture, + ScrubberLocation location) => + ScrubInlineDateTimes(format, culture); + + /// + /// Replace inline s with a placeholder. + /// + public static void ScrubInlineDateTimeOffsets( + [StringSyntax(StringSyntaxAttribute.DateTimeFormat)] string format, + Culture? culture = null) + { + foreach (var scrubber in DateMatchers.DateTimeOffsets(format, culture)) + { + AddScrubber(scrubber); + } + } + + /// + /// Replace inline s with a placeholder. + /// + [Obsolete(locationObsolete)] public static void ScrubInlineDateTimeOffsets( [StringSyntax(StringSyntaxAttribute.DateTimeFormat)] string format, - Culture? culture = null, - ScrubberLocation location = ScrubberLocation.First) => - AddScrubber( - DateScrubber.BuildDateTimeOffsetScrubber(format, culture), - location); + Culture? culture, + ScrubberLocation location) => + ScrubInlineDateTimeOffsets(format, culture); #if NET6_0_OR_GREATER @@ -106,66 +148,90 @@ public static void ScrubInlineDateTimeOffsets( /// public static void ScrubInlineDates( [StringSyntax(StringSyntaxAttribute.DateOnlyFormat)] string format, - Culture? culture = null, - ScrubberLocation location = ScrubberLocation.First) => - AddScrubber( - DateScrubber.BuildDateScrubber(format, culture), - location); + Culture? culture = null) + { + foreach (var scrubber in DateMatchers.Dates(format, culture)) + { + AddScrubber(scrubber); + } + } + + /// + /// Replace inline s with a placeholder. + /// + [Obsolete(locationObsolete)] + public static void ScrubInlineDates( + [StringSyntax(StringSyntaxAttribute.DateOnlyFormat)] string format, + Culture? culture, + ScrubberLocation location) => + ScrubInlineDates(format, culture); #endif /// /// Replace inline s with a placeholder. /// - public static void ScrubInlineGuids(ScrubberLocation location = ScrubberLocation.First) - { - InnerVerifier.ThrowIfVerifyHasBeenRun(); - AddScrubber(GuidScrubber.ReplaceGuids, location); - } + public static void ScrubInlineGuids() => + AddScrubber(GuidMatcher.Instance); + + /// + /// Replace inline s with a placeholder. + /// + [Obsolete(locationObsolete)] + public static void ScrubInlineGuids(ScrubberLocation location) => + ScrubInlineGuids(); /// /// Scrub lines with an optional replace. /// can return the input to ignore the line, or return a different string to replace it. /// - public static void ScrubLinesWithReplace(Func replaceLine, ScrubberLocation location = ScrubberLocation.First) - { - InnerVerifier.ThrowIfVerifyHasBeenRun(); - AddScrubber(_ => _.ReplaceLines(replaceLine), location); - } + public static void ScrubLinesWithReplace(Func replaceLine) => + AddScrubber(Scrubber.ReplaceLines(replaceLine)); + + /// + /// Scrub lines with an optional replace. + /// can return the input to ignore the line, or return a different string to replace it. + /// + [Obsolete(locationObsolete)] + public static void ScrubLinesWithReplace(Func replaceLine, ScrubberLocation location) => + ScrubLinesWithReplace(replaceLine); /// /// Remove any lines containing any of from the test results. /// - public static void ScrubLinesContaining(params string[] stringToMatch) - { - InnerVerifier.ThrowIfVerifyHasBeenRun(); - ScrubLinesContaining(ScrubberLocation.First, stringToMatch); - } + public static void ScrubLinesContaining(params string[] stringToMatch) => + ScrubLinesContaining(StringComparison.OrdinalIgnoreCase, stringToMatch); /// /// Remove any lines containing any of from the test results. /// - public static void ScrubLinesContaining(ScrubberLocation location = ScrubberLocation.First, params string[] stringToMatch) - { - InnerVerifier.ThrowIfVerifyHasBeenRun(); - ScrubLinesContaining(StringComparison.OrdinalIgnoreCase, location, stringToMatch); - } + [Obsolete(locationObsolete)] + public static void ScrubLinesContaining(ScrubberLocation location, params string[] stringToMatch) => + ScrubLinesContaining(StringComparison.OrdinalIgnoreCase, stringToMatch); /// /// Remove the from the test results. /// - public static void ScrubMachineName(ScrubberLocation location = ScrubberLocation.First) - { - InnerVerifier.ThrowIfVerifyHasBeenRun(); - AddScrubber(UserMachineScrubber.Machine, location); - } + public static void ScrubMachineName() => + AddScrubber(UserMachineScrubber.MachineScrubber()); + + /// + /// Remove the from the test results. + /// + [Obsolete(locationObsolete)] + public static void ScrubMachineName(ScrubberLocation location) => + ScrubMachineName(); /// /// Remove the from the test results. /// - public static void ScrubUserName(ScrubberLocation location = ScrubberLocation.First) - { - InnerVerifier.ThrowIfVerifyHasBeenRun(); - AddScrubber(UserMachineScrubber.User, location); - } -} \ No newline at end of file + public static void ScrubUserName() => + AddScrubber(UserMachineScrubber.UserScrubber()); + + /// + /// Remove the from the test results. + /// + [Obsolete(locationObsolete)] + public static void ScrubUserName(ScrubberLocation location) => + ScrubUserName(); +} diff --git a/src/Verify/Serialization/Scrubbers/VerifySettings_ExtensionMappedInstanceScrubbers.cs b/src/Verify/Serialization/Scrubbers/VerifySettings_ExtensionMappedInstanceScrubbers.cs index 192538788d..6c5e69764f 100644 --- a/src/Verify/Serialization/Scrubbers/VerifySettings_ExtensionMappedInstanceScrubbers.cs +++ b/src/Verify/Serialization/Scrubbers/VerifySettings_ExtensionMappedInstanceScrubbers.cs @@ -1,9 +1,27 @@ -namespace VerifyTests; +namespace VerifyTests; public partial class VerifySettings { internal Dictionary>>>? ExtensionMappedInstanceScrubbers = []; + internal Dictionary>? ExtensionMappedInstanceSpanScrubbers; + + /// + /// Add a that applies to verified files with a matching extension. + /// + public void AddScrubber(string extension, Scrubber scrubber) + { + Ensure.NotNull(scrubber); + ExtensionMappedInstanceSpanScrubbers ??= []; + + if (!ExtensionMappedInstanceSpanScrubbers.TryGetValue(extension, out var values)) + { + ExtensionMappedInstanceSpanScrubbers[extension] = values = []; + } + + values.Add(scrubber); + } + /// /// Modify the resulting test content using custom code. /// @@ -43,55 +61,100 @@ public void AddScrubber(string extension, Action /// Remove the from the test results. /// - public void ScrubMachineName(string extension, ScrubberLocation location = ScrubberLocation.First) => - AddScrubber(extension, UserMachineScrubber.Machine, location); + public void ScrubMachineName(string extension) => + AddScrubber(extension, UserMachineScrubber.MachineScrubber()); + + /// + /// Remove the from the test results. + /// + [Obsolete(locationObsolete)] + public void ScrubMachineName(string extension, ScrubberLocation location) => + ScrubMachineName(extension); + + /// + /// Remove the from the test results. + /// + public void ScrubUserName(string extension) => + AddScrubber(extension, UserMachineScrubber.UserScrubber()); /// /// Remove the from the test results. /// - public void ScrubUserName(string extension, ScrubberLocation location = ScrubberLocation.First) => - AddScrubber(extension, UserMachineScrubber.User, location); + [Obsolete(locationObsolete)] + public void ScrubUserName(string extension, ScrubberLocation location) => + ScrubUserName(extension); /// /// Remove any lines containing any of from the test results. /// public void ScrubLinesContaining(string extension, StringComparison comparison, params string[] stringToMatch) => - ScrubLinesContaining(extension, comparison, ScrubberLocation.First, stringToMatch); + AddScrubber(extension, Scrubber.RemoveLinesContaining(comparison, stringToMatch)); /// /// Remove any lines containing any of from the test results. /// + [Obsolete(locationObsolete)] public void ScrubLinesContaining(string extension, StringComparison comparison, ScrubberLocation location, params string[] stringToMatch) => - AddScrubber(extension, _ => _.RemoveLinesContaining(comparison, stringToMatch), location); + ScrubLinesContaining(extension, comparison, stringToMatch); /// /// Replace inline s with a placeholder. /// - public void ScrubInlineGuids(string extension, ScrubberLocation location = ScrubberLocation.First) => - AddScrubber(extension, GuidScrubber.ReplaceGuids, location); + public void ScrubInlineGuids(string extension) => + AddScrubber(extension, GuidMatcher.Instance); + + /// + /// Replace inline s with a placeholder. + /// + [Obsolete(locationObsolete)] + public void ScrubInlineGuids(string extension, ScrubberLocation location) => + ScrubInlineGuids(extension); /// /// Remove any lines matching from the test results. /// - public void ScrubLines(string extension, Func removeLine, ScrubberLocation location = ScrubberLocation.First) => - AddScrubber(extension, _ => _.FilterLines(removeLine), location); + public void ScrubLines(string extension, Func removeLine) => + AddScrubber(extension, Scrubber.RemoveLines(removeLine)); + + /// + /// Remove any lines matching from the test results. + /// + [Obsolete(locationObsolete)] + public void ScrubLines(string extension, Func removeLine, ScrubberLocation location) => + ScrubLines(extension, removeLine); /// /// Scrub lines with an optional replace. /// can return the input to ignore the line, or return a different string to replace it. /// - public void ScrubLinesWithReplace(string extension, Func replaceLine, ScrubberLocation location = ScrubberLocation.First) => - AddScrubber(extension, _ => _.ReplaceLines(replaceLine), location); + public void ScrubLinesWithReplace(string extension, Func replaceLine) => + AddScrubber(extension, Scrubber.ReplaceLines(replaceLine)); + + /// + /// Scrub lines with an optional replace. + /// can return the input to ignore the line, or return a different string to replace it. + /// + [Obsolete(locationObsolete)] + public void ScrubLinesWithReplace(string extension, Func replaceLine, ScrubberLocation location) => + ScrubLinesWithReplace(extension, replaceLine); + + /// + /// Remove any lines containing only whitespace from the test results. + /// + public void ScrubEmptyLines(string extension) => + AddScrubber(extension, Scrubber.RemoveEmptyLines()); /// /// Remove any lines containing only whitespace from the test results. /// - public void ScrubEmptyLines(string extension, ScrubberLocation location = ScrubberLocation.First) => - AddScrubber(extension, _ => _.RemoveEmptyLines(), location); + [Obsolete(locationObsolete)] + public void ScrubEmptyLines(string extension, ScrubberLocation location) => + ScrubEmptyLines(extension); /// /// Remove any lines containing any of from the test results. /// - public void ScrubLinesContaining(string extension, ScrubberLocation location = ScrubberLocation.First, params string[] stringToMatch) => - ScrubLinesContaining(extension, StringComparison.OrdinalIgnoreCase, location, stringToMatch); -} \ No newline at end of file + [Obsolete(locationObsolete)] + public void ScrubLinesContaining(string extension, ScrubberLocation location, params string[] stringToMatch) => + ScrubLinesContaining(extension, StringComparison.OrdinalIgnoreCase, stringToMatch); +} diff --git a/src/Verify/Serialization/Scrubbers/VerifySettings_InstanceScrubbers.cs b/src/Verify/Serialization/Scrubbers/VerifySettings_InstanceScrubbers.cs index c6a79b0906..06d4b8c528 100644 --- a/src/Verify/Serialization/Scrubbers/VerifySettings_InstanceScrubbers.cs +++ b/src/Verify/Serialization/Scrubbers/VerifySettings_InstanceScrubbers.cs @@ -1,9 +1,23 @@ -namespace VerifyTests; +namespace VerifyTests; public partial class VerifySettings { internal List>>? InstanceScrubbers = []; + internal List? InstanceSpanScrubbers; + + const string locationObsolete = "ScrubberLocation is ignored; span scrubber ordering is engine determined. Use the overload without ScrubberLocation."; + + /// + /// Add a . + /// + public void AddScrubber(Scrubber scrubber) + { + Ensure.NotNull(scrubber); + InstanceSpanScrubbers ??= []; + InstanceSpanScrubbers.Add(scrubber); + } + internal bool ScrubbersEnabled { get; private set; } = true; /// @@ -14,14 +28,28 @@ public partial class VerifySettings /// /// Remove the from the test results. /// - public void ScrubMachineName(ScrubberLocation location = ScrubberLocation.First) => - AddScrubber(UserMachineScrubber.Machine, location); + public void ScrubMachineName() => + AddScrubber(UserMachineScrubber.MachineScrubber()); + + /// + /// Remove the from the test results. + /// + [Obsolete(locationObsolete)] + public void ScrubMachineName(ScrubberLocation location) => + ScrubMachineName(); + + /// + /// Remove the from the test results. + /// + public void ScrubUserName() => + AddScrubber(UserMachineScrubber.UserScrubber()); /// /// Remove the from the test results. /// - public void ScrubUserName(ScrubberLocation location = ScrubberLocation.First) => - AddScrubber(UserMachineScrubber.User, location); + [Obsolete(locationObsolete)] + public void ScrubUserName(ScrubberLocation location) => + ScrubUserName(); /// /// Modify the resulting test content using custom code. @@ -64,65 +92,95 @@ public void AddScrubber(Action from the test results. /// public void ScrubLinesContaining(StringComparison comparison, params string[] stringToMatch) => - ScrubLinesContaining(comparison, ScrubberLocation.First, stringToMatch); + AddScrubber(Scrubber.RemoveLinesContaining(comparison, stringToMatch)); /// /// Remove any lines containing any of from the test results. /// + [Obsolete(locationObsolete)] public void ScrubLinesContaining(StringComparison comparison, ScrubberLocation location, params string[] stringToMatch) => - AddScrubber(_ => _.RemoveLinesContaining(comparison, stringToMatch), location); + ScrubLinesContaining(comparison, stringToMatch); /// /// Replace inline s with a placeholder. /// - public void ScrubInlineGuids(ScrubberLocation location = ScrubberLocation.First) + public void ScrubInlineGuids() { if (serialization.ScrubGuids == false) { throw new("ScrubGuids is disabled. Call .ScrubGuids() before calling .ScrubInlineGuids()."); } - AddScrubber(GuidScrubber.ReplaceGuids, location); + AddScrubber(GuidMatcher.Instance); } + /// + /// Replace inline s with a placeholder. + /// + [Obsolete(locationObsolete)] + public void ScrubInlineGuids(ScrubberLocation location) => + ScrubInlineGuids(); + /// /// Replace inline s with a placeholder. /// public void ScrubInlineDateTimes( [StringSyntax(StringSyntaxAttribute.DateTimeFormat)] string format, - Culture? culture = null, - ScrubberLocation location = ScrubberLocation.First) + Culture? culture = null) { if (serialization.ScrubDateTimes == false) { throw new("ScrubDateTimes is disabled. Call .ScrubDateTimes() before calling .ScrubInlineDateTimes()."); } - AddScrubber( - DateScrubber.BuildDateTimeScrubber(format, culture), - location); + foreach (var scrubber in DateMatchers.DateTimes(format, culture)) + { + AddScrubber(scrubber); + } } /// /// Replace inline s with a placeholder. /// + [Obsolete(locationObsolete)] + public void ScrubInlineDateTimes( + [StringSyntax(StringSyntaxAttribute.DateTimeFormat)] + string format, + Culture? culture, + ScrubberLocation location) => + ScrubInlineDateTimes(format, culture); + + /// + /// Replace inline s with a placeholder. + /// public void ScrubInlineDateTimeOffsets( [StringSyntax(StringSyntaxAttribute.DateTimeFormat)] string format, - Culture? culture = null, - ScrubberLocation location = ScrubberLocation.First) + Culture? culture = null) { if (serialization.ScrubDateTimes == false) { throw new("ScrubDateTimes is disabled. Call .ScrubDateTimes() before calling .ScrubInlineDateTimeOffsets()."); } - AddScrubber( - DateScrubber.BuildDateTimeOffsetScrubber(format, culture), - location); + foreach (var scrubber in DateMatchers.DateTimeOffsets(format, culture)) + { + AddScrubber(scrubber); + } } + /// + /// Replace inline s with a placeholder. + /// + [Obsolete(locationObsolete)] + public void ScrubInlineDateTimeOffsets( + [StringSyntax(StringSyntaxAttribute.DateTimeFormat)] + string format, + Culture? culture, + ScrubberLocation location) => + ScrubInlineDateTimeOffsets(format, culture); + #if NET6_0_OR_GREATER /// @@ -130,49 +188,82 @@ public void ScrubInlineDateTimeOffsets( /// public void ScrubInlineDates( [StringSyntax(StringSyntaxAttribute.DateOnlyFormat)] string format, - Culture? culture = null, - ScrubberLocation location = ScrubberLocation.First) + Culture? culture = null) { if (serialization.ScrubDateTimes == false) { throw new("ScrubDateTimes is disabled. Call .ScrubDateTimes() before calling .ScrubInlineDates()."); } - AddScrubber( - DateScrubber.BuildDateScrubber(format, culture), - location); + foreach (var scrubber in DateMatchers.Dates(format, culture)) + { + AddScrubber(scrubber); + } } + /// + /// Replace inline s with a placeholder. + /// + [Obsolete(locationObsolete)] + public void ScrubInlineDates( + [StringSyntax(StringSyntaxAttribute.DateOnlyFormat)] string format, + Culture? culture, + ScrubberLocation location) => + ScrubInlineDates(format, culture); + #endif /// /// Remove any lines matching from the test results. /// - public void ScrubLines(Func removeLine, ScrubberLocation location = ScrubberLocation.First) => - AddScrubber(_ => _.FilterLines(removeLine), location); + public void ScrubLines(Func removeLine) => + AddScrubber(Scrubber.RemoveLines(removeLine)); + + /// + /// Remove any lines matching from the test results. + /// + [Obsolete(locationObsolete)] + public void ScrubLines(Func removeLine, ScrubberLocation location) => + ScrubLines(removeLine); + + /// + /// Scrub lines with an optional replace. + /// can return the input to ignore the line, or return a different string to replace it. + /// + public void ScrubLinesWithReplace(Func replaceLine) => + AddScrubber(Scrubber.ReplaceLines(replaceLine)); /// /// Scrub lines with an optional replace. /// can return the input to ignore the line, or return a different string to replace it. /// - public void ScrubLinesWithReplace(Func replaceLine, ScrubberLocation location = ScrubberLocation.First) => - AddScrubber(_ => _.ReplaceLines(replaceLine), location); + [Obsolete(locationObsolete)] + public void ScrubLinesWithReplace(Func replaceLine, ScrubberLocation location) => + ScrubLinesWithReplace(replaceLine); + + /// + /// Remove any lines containing only whitespace from the test results. + /// + public void ScrubEmptyLines() => + AddScrubber(Scrubber.RemoveEmptyLines()); /// /// Remove any lines containing only whitespace from the test results. /// - public void ScrubEmptyLines(ScrubberLocation location = ScrubberLocation.First) => - AddScrubber(_ => _.RemoveEmptyLines(), location); + [Obsolete(locationObsolete)] + public void ScrubEmptyLines(ScrubberLocation location) => + ScrubEmptyLines(); /// /// Remove any lines containing any of from the test results. /// public void ScrubLinesContaining(params string[] stringToMatch) => - ScrubLinesContaining(ScrubberLocation.First, stringToMatch); + ScrubLinesContaining(StringComparison.OrdinalIgnoreCase, stringToMatch); /// /// Remove any lines containing any of from the test results. /// - public void ScrubLinesContaining(ScrubberLocation location = ScrubberLocation.First, params string[] stringToMatch) => - ScrubLinesContaining(StringComparison.OrdinalIgnoreCase, location, stringToMatch); -} \ No newline at end of file + [Obsolete(locationObsolete)] + public void ScrubLinesContaining(ScrubberLocation location, params string[] stringToMatch) => + ScrubLinesContaining(StringComparison.OrdinalIgnoreCase, stringToMatch); +} diff --git a/src/Verify/Serialization/VerifierSettings.cs b/src/Verify/Serialization/VerifierSettings.cs index 93e2e96b9a..ae9799c20a 100644 --- a/src/Verify/Serialization/VerifierSettings.cs +++ b/src/Verify/Serialization/VerifierSettings.cs @@ -167,6 +167,10 @@ internal static void Reset() addAttachments = true; excludedTargets = null; GlobalScrubbers.Clear(); + ExtensionMappedGlobalScrubbers.Clear(); + GlobalSpanScrubbers.Clear(); + ExtensionMappedGlobalSpanScrubbers.Clear(); + EngineScrubberSet.InvalidateGlobalCache(); GlobalIgnoredParameters = null; GlobalIgnoreConstructorParameters = false; } diff --git a/src/Verify/SettingsTask_Scrubbing.cs b/src/Verify/SettingsTask_Scrubbing.cs index 947666d540..b9edc406a4 100644 --- a/src/Verify/SettingsTask_Scrubbing.cs +++ b/src/Verify/SettingsTask_Scrubbing.cs @@ -1,7 +1,9 @@ -namespace VerifyTests; +namespace VerifyTests; public partial class SettingsTask { + const string locationObsolete = "ScrubberLocation is ignored; span scrubber ordering is engine determined. Use the overload without ScrubberLocation."; + /// [Pure] public SettingsTask DisableScrubbers() @@ -10,6 +12,22 @@ public SettingsTask DisableScrubbers() return this; } + /// + [Pure] + public SettingsTask AddScrubber(Scrubber scrubber) + { + CurrentSettings.AddScrubber(scrubber); + return this; + } + + /// + [Pure] + public SettingsTask AddScrubber(string extension, Scrubber scrubber) + { + CurrentSettings.AddScrubber(extension, scrubber); + return this; + } + /// [Pure] public SettingsTask AddScrubber(Action scrubber, ScrubberLocation location = ScrubberLocation.First) @@ -26,14 +44,20 @@ public SettingsTask AddScrubber(string extension, Action scrubber return this; } - /// + /// [Pure] - public SettingsTask ScrubInlineGuids(ScrubberLocation location = ScrubberLocation.First) + public SettingsTask ScrubInlineGuids() { - CurrentSettings.ScrubInlineGuids(location); + CurrentSettings.ScrubInlineGuids(); return this; } + /// + [Obsolete(locationObsolete)] + [Pure] + public SettingsTask ScrubInlineGuids(ScrubberLocation location) => + ScrubInlineGuids(); + /// [Pure] public SettingsTask ScrubGuids() @@ -50,14 +74,20 @@ public SettingsTask ScrubNumericIds() return this; } - /// + /// [Pure] - public SettingsTask ScrubInlineGuids(string extension, ScrubberLocation location = ScrubberLocation.First) + public SettingsTask ScrubInlineGuids(string extension) { - CurrentSettings.ScrubInlineGuids(extension, location); + CurrentSettings.ScrubInlineGuids(extension); return this; } + /// + [Obsolete(locationObsolete)] + [Pure] + public SettingsTask ScrubInlineGuids(string extension, ScrubberLocation location) => + ScrubInlineGuids(extension); + /// [Pure] public SettingsTask DisableDateCounting() @@ -74,75 +104,123 @@ public SettingsTask ScrubDateTimes() return this; } - /// + /// [Pure] public SettingsTask ScrubInlineDateTimes( [StringSyntax(StringSyntaxAttribute.DateTimeFormat)] string format, - Culture? culture = null, - ScrubberLocation location = ScrubberLocation.First) + Culture? culture = null) { - CurrentSettings.ScrubInlineDateTimes(format, culture, location); + CurrentSettings.ScrubInlineDateTimes(format, culture); return this; } - /// + /// + [Obsolete(locationObsolete)] + [Pure] + public SettingsTask ScrubInlineDateTimes( + [StringSyntax(StringSyntaxAttribute.DateTimeFormat)] string format, + Culture? culture, + ScrubberLocation location) => + ScrubInlineDateTimes(format, culture); + + /// [Pure] public SettingsTask ScrubInlineDateTimeOffsets( [StringSyntax(StringSyntaxAttribute.DateTimeFormat)] string format, - Culture? culture = null, - ScrubberLocation location = ScrubberLocation.First) + Culture? culture = null) { - CurrentSettings.ScrubInlineDateTimeOffsets(format, culture, location); + CurrentSettings.ScrubInlineDateTimeOffsets(format, culture); return this; } + /// + [Obsolete(locationObsolete)] + [Pure] + public SettingsTask ScrubInlineDateTimeOffsets( + [StringSyntax(StringSyntaxAttribute.DateTimeFormat)] string format, + Culture? culture, + ScrubberLocation location) => + ScrubInlineDateTimeOffsets(format, culture); + #if NET6_0_OR_GREATER - /// + /// [Pure] public SettingsTask ScrubInlineDates( [StringSyntax(StringSyntaxAttribute.DateOnlyFormat)] string format, - Culture? culture = null, - ScrubberLocation location = ScrubberLocation.First) + Culture? culture = null) { - CurrentSettings.ScrubInlineDates(format, culture, location); + CurrentSettings.ScrubInlineDates(format, culture); return this; } + /// + [Obsolete(locationObsolete)] + [Pure] + public SettingsTask ScrubInlineDates( + [StringSyntax(StringSyntaxAttribute.DateOnlyFormat)] string format, + Culture? culture, + ScrubberLocation location) => + ScrubInlineDates(format, culture); + #endif - /// + /// [Pure] - public SettingsTask ScrubMachineName(ScrubberLocation location = ScrubberLocation.First) + public SettingsTask ScrubMachineName() { - CurrentSettings.ScrubMachineName(location); + CurrentSettings.ScrubMachineName(); return this; } - /// + /// + [Obsolete(locationObsolete)] [Pure] - public SettingsTask ScrubMachineName(string extension, ScrubberLocation location = ScrubberLocation.First) + public SettingsTask ScrubMachineName(ScrubberLocation location) => + ScrubMachineName(); + + /// + [Pure] + public SettingsTask ScrubMachineName(string extension) { - CurrentSettings.ScrubMachineName(extension, location); + CurrentSettings.ScrubMachineName(extension); return this; } - /// + /// + [Obsolete(locationObsolete)] + [Pure] + public SettingsTask ScrubMachineName(string extension, ScrubberLocation location) => + ScrubMachineName(extension); + + /// [Pure] - public SettingsTask ScrubUserName(ScrubberLocation location = ScrubberLocation.First) + public SettingsTask ScrubUserName() { - CurrentSettings.ScrubUserName(location); + CurrentSettings.ScrubUserName(); return this; } - /// + /// + [Obsolete(locationObsolete)] [Pure] - public SettingsTask ScrubUserName(string extension,ScrubberLocation location = ScrubberLocation.First) + public SettingsTask ScrubUserName(ScrubberLocation location) => + ScrubUserName(); + + /// + [Pure] + public SettingsTask ScrubUserName(string extension) { - CurrentSettings.ScrubUserName(extension, location); + CurrentSettings.ScrubUserName(extension); return this; } + /// + [Obsolete(locationObsolete)] + [Pure] + public SettingsTask ScrubUserName(string extension, ScrubberLocation location) => + ScrubUserName(extension); + /// [Pure] public SettingsTask ScrubLinesContaining(StringComparison comparison, params string[] stringToMatch) @@ -159,70 +237,102 @@ public SettingsTask ScrubLinesContaining(string extension, StringComparison comp return this; } - /// + /// + [Obsolete(locationObsolete)] [Pure] - public SettingsTask ScrubLinesContaining(StringComparison comparison, ScrubberLocation location = ScrubberLocation.First, params string[] stringToMatch) - { - CurrentSettings.ScrubLinesContaining(comparison, location, stringToMatch); - return this; - } + public SettingsTask ScrubLinesContaining(StringComparison comparison, ScrubberLocation location, params string[] stringToMatch) => + ScrubLinesContaining(comparison, stringToMatch); - /// + /// + [Obsolete(locationObsolete)] [Pure] - public SettingsTask ScrubLinesContaining(string extension, StringComparison comparison, ScrubberLocation location = ScrubberLocation.First, params string[] stringToMatch) - { - CurrentSettings.ScrubLinesContaining(extension, comparison, location, stringToMatch); - return this; - } + public SettingsTask ScrubLinesContaining(string extension, StringComparison comparison, ScrubberLocation location, params string[] stringToMatch) => + ScrubLinesContaining(extension, comparison, stringToMatch); - /// + /// [Pure] - public SettingsTask ScrubLines(Func removeLine, ScrubberLocation location = ScrubberLocation.First) + public SettingsTask ScrubLines(Func removeLine) { - CurrentSettings.ScrubLines(removeLine, location); + CurrentSettings.ScrubLines(removeLine); return this; } - /// + /// + [Obsolete(locationObsolete)] [Pure] - public SettingsTask ScrubLines(string extension, Func removeLine, ScrubberLocation location = ScrubberLocation.First) + public SettingsTask ScrubLines(Func removeLine, ScrubberLocation location) => + ScrubLines(removeLine); + + /// + [Pure] + public SettingsTask ScrubLines(string extension, Func removeLine) { - CurrentSettings.ScrubLines(extension, removeLine, location); + CurrentSettings.ScrubLines(extension, removeLine); return this; } - /// + /// + [Obsolete(locationObsolete)] + [Pure] + public SettingsTask ScrubLines(string extension, Func removeLine, ScrubberLocation location) => + ScrubLines(extension, removeLine); + + /// [Pure] - public SettingsTask ScrubLinesWithReplace(Func replaceLine, ScrubberLocation location = ScrubberLocation.First) + public SettingsTask ScrubLinesWithReplace(Func replaceLine) { - CurrentSettings.ScrubLinesWithReplace(replaceLine, location); + CurrentSettings.ScrubLinesWithReplace(replaceLine); return this; } - /// + /// + [Obsolete(locationObsolete)] + [Pure] + public SettingsTask ScrubLinesWithReplace(Func replaceLine, ScrubberLocation location) => + ScrubLinesWithReplace(replaceLine); + + /// [Pure] - public SettingsTask ScrubLinesWithReplace(string extension, Func replaceLine, ScrubberLocation location = ScrubberLocation.First) + public SettingsTask ScrubLinesWithReplace(string extension, Func replaceLine) { - CurrentSettings.ScrubLinesWithReplace(extension, replaceLine, location); + CurrentSettings.ScrubLinesWithReplace(extension, replaceLine); return this; } - /// + /// + [Obsolete(locationObsolete)] + [Pure] + public SettingsTask ScrubLinesWithReplace(string extension, Func replaceLine, ScrubberLocation location) => + ScrubLinesWithReplace(extension, replaceLine); + + /// [Pure] - public SettingsTask ScrubEmptyLines(ScrubberLocation location = ScrubberLocation.First) + public SettingsTask ScrubEmptyLines() { - CurrentSettings.ScrubEmptyLines(location); + CurrentSettings.ScrubEmptyLines(); return this; } - /// + /// + [Obsolete(locationObsolete)] + [Pure] + public SettingsTask ScrubEmptyLines(ScrubberLocation location) => + ScrubEmptyLines(); + + /// [Pure] - public SettingsTask ScrubEmptyLines(string extension, ScrubberLocation location = ScrubberLocation.First) + public SettingsTask ScrubEmptyLines(string extension) { - CurrentSettings.ScrubEmptyLines(extension, location); + CurrentSettings.ScrubEmptyLines(extension); return this; } + /// + [Obsolete(locationObsolete)] + [Pure] + public SettingsTask ScrubEmptyLines(string extension, ScrubberLocation location) => + ScrubEmptyLines(extension); + /// [Pure] public SettingsTask ScrubLinesContaining(params string[] stringToMatch) @@ -231,19 +341,18 @@ public SettingsTask ScrubLinesContaining(params string[] stringToMatch) return this; } - /// + /// + [Obsolete(locationObsolete)] [Pure] - public SettingsTask ScrubLinesContaining(ScrubberLocation location = ScrubberLocation.First, params string[] stringToMatch) - { - CurrentSettings.ScrubLinesContaining(location, stringToMatch); - return this; - } + public SettingsTask ScrubLinesContaining(ScrubberLocation location, params string[] stringToMatch) => + ScrubLinesContaining(stringToMatch); - /// + /// + [Obsolete(locationObsolete)] [Pure] - public SettingsTask ScrubLinesContaining(string extension, ScrubberLocation location = ScrubberLocation.First, params string[] stringToMatch) + public SettingsTask ScrubLinesContaining(string extension, ScrubberLocation location, params string[] stringToMatch) { - CurrentSettings.ScrubLinesContaining(extension, location, stringToMatch); + CurrentSettings.ScrubLinesContaining(extension, StringComparison.OrdinalIgnoreCase, stringToMatch); return this; } -} \ No newline at end of file +} diff --git a/src/Verify/TempDirectory.cs b/src/Verify/TempDirectory.cs index 4b56fa14aa..edc754de9c 100644 --- a/src/Verify/TempDirectory.cs +++ b/src/Verify/TempDirectory.cs @@ -63,22 +63,43 @@ public static void Init() }, after: () => asyncPaths.Value = null); - VerifierSettings.GlobalScrubbers.Add((scrubber, _, _) => - { - var pathsValue = asyncPaths.Value; - if (pathsValue == null) - { - return; - } - - lock (pathsLock) - { - foreach (var path in pathsValue) + // All temp directory paths start with RootDirectory, so segments shorter + // than that can be skipped without invoking the matcher + VerifierSettings.AddScrubber( + Scrubber.Match( + (CharSpan segment, Counter _, IReadOnlyDictionary _, out int index, out int length, out string? replacement) => { - scrubber.Replace(path, "{TempDirectory}"); - } - } - }); + index = -1; + length = 0; + replacement = "{TempDirectory}"; + var pathsValue = asyncPaths.Value; + if (pathsValue == null) + { + return false; + } + + lock (pathsLock) + { + foreach (var path in pathsValue) + { + var found = segment.IndexOf(path.AsSpan(), StringComparison.Ordinal); + if (found < 0) + { + continue; + } + + if (index < 0 || + found < index) + { + index = found; + length = path.Length; + } + } + } + + return index >= 0; + }, + minLength: RootDirectory.Length)); Cleanup(); } diff --git a/src/Verify/TempFile.cs b/src/Verify/TempFile.cs index 612745f375..f5972fb649 100644 --- a/src/Verify/TempFile.cs +++ b/src/Verify/TempFile.cs @@ -65,22 +65,43 @@ public static void Init() }, after: () => asyncPaths.Value = null); - VerifierSettings.GlobalScrubbers.Add((scrubber, _, _) => - { - var pathsValue = asyncPaths.Value; - if (pathsValue == null) - { - return; - } - - lock (pathsLock) - { - foreach (var path in pathsValue) + // All temp file paths start with RootDirectory, so segments shorter than + // that can be skipped without invoking the matcher + VerifierSettings.AddScrubber( + Scrubber.Match( + (CharSpan segment, Counter _, IReadOnlyDictionary _, out int index, out int length, out string? replacement) => { - scrubber.Replace(path, "{TempFile}"); - } - } - }); + index = -1; + length = 0; + replacement = "{TempFile}"; + var pathsValue = asyncPaths.Value; + if (pathsValue == null) + { + return false; + } + + lock (pathsLock) + { + foreach (var path in pathsValue) + { + var found = segment.IndexOf(path.AsSpan(), StringComparison.Ordinal); + if (found < 0) + { + continue; + } + + if (index < 0 || + found < index) + { + index = found; + length = path.Length; + } + } + } + + return index >= 0; + }, + minLength: RootDirectory.Length)); Cleanup(); } diff --git a/src/Verify/Verifier/InnerVerifier_Xml.cs b/src/Verify/Verifier/InnerVerifier_Xml.cs index b04b44406f..b36d20ab40 100644 --- a/src/Verify/Verifier/InnerVerifier_Xml.cs +++ b/src/Verify/Verifier/InnerVerifier_Xml.cs @@ -103,13 +103,12 @@ Task VerifyXml(XContainer? target) string ConvertValue(string value) { - var span = value.AsSpan(); - if (counter.TryConvert(span, out var result)) + if (counter.TryConvert(value.AsSpan(), out var result)) { return result; } - return ApplyScrubbers.ApplyForPropertyValue(span, settings, counter); + return ApplyScrubbers.ApplyForPropertyValue(value, settings, counter); } void ScrubAttributes(XElement node, SerializationSettings serialization) @@ -132,14 +131,14 @@ void ScrubAttributes(XElement node, SerializationSettings serialization) continue; } - var span = attribute.Value.AsSpan(); - if (counter.TryConvert(span, out var result)) + var value = attribute.Value; + if (counter.TryConvert(value.AsSpan(), out var result)) { attribute.Value = result; } else { - attribute.Value = ApplyScrubbers.ApplyForPropertyValue(span, settings, counter); + attribute.Value = ApplyScrubbers.ApplyForPropertyValue(value, settings, counter); } } } diff --git a/src/Verify/VerifySettings.cs b/src/Verify/VerifySettings.cs index 19c8f964f1..d40404ac31 100644 --- a/src/Verify/VerifySettings.cs +++ b/src/Verify/VerifySettings.cs @@ -36,6 +36,18 @@ public VerifySettings(VerifySettings? settings) .ToDictionary(_ => _.Key, _ => _.Value.ToList()); } + if (settings.InstanceSpanScrubbers != null) + { + InstanceSpanScrubbers = [..settings.InstanceSpanScrubbers]; + } + + if (settings.ExtensionMappedInstanceSpanScrubbers != null) + { + // Deep copy: the inner lists are mutated in place by AddScrubber. + ExtensionMappedInstanceSpanScrubbers = settings.ExtensionMappedInstanceSpanScrubbers + .ToDictionary(_ => _.Key, _ => _.Value.ToList()); + } + diffEnabled = settings.diffEnabled; MethodName = settings.MethodName; TypeName = settings.TypeName; From 710e3c2764e084b9a8bb3b99cb1580d402448843 Mon Sep 17 00:00:00 2001 From: Simon Cropp Date: Sat, 11 Jul 2026 14:36:11 +1000 Subject: [PATCH 02/33] . --- docs/mdsource/scrubbers.source.md | 2 ++ docs/scrubbers.md | 2 ++ src/ApplyScrubbersTests/LinePhaseTests.cs | 36 +++++++++++++++++++ src/Benchmarks/FilterLinesBenchmarks.cs | 28 +++++++++++++++ .../Serialization/Scrubbers/Scrubber.cs | 8 +++++ ...Settings_ExtensionMappedGlobalScrubbers.cs | 20 +++++++++++ .../VerifierSettings_GlobalScrubbers.cs | 20 +++++++++++ ...ttings_ExtensionMappedInstanceScrubbers.cs | 20 +++++++++++ .../VerifySettings_InstanceScrubbers.cs | 20 +++++++++++ src/Verify/SettingsTask_Scrubbing.cs | 36 +++++++++++++++++++ src/VerifyCore.slnf | 1 + 11 files changed, 193 insertions(+) diff --git a/docs/mdsource/scrubbers.source.md b/docs/mdsource/scrubbers.source.md index 2134bcd2d0..edc267407b 100644 --- a/docs/mdsource/scrubbers.source.md +++ b/docs/mdsource/scrubbers.source.md @@ -19,6 +19,8 @@ A `Scrubber` is created via static factory methods and registered via `AddScrubb * `Scrubber.Match(matcher, minLength, maxLength)`: custom search logic. The matcher locates the next match within a segment. * `Scrubber.RemoveLinesContaining(...)`, `Scrubber.RemoveLines(...)`, `Scrubber.ReplaceLines(...)`, `Scrubber.RemoveEmptyLines()`: line scoped scrubbers. +`ScrubLines` and `ScrubLinesWithReplace` (and the corresponding `Scrubber` factories) also accept span based delegates (`LineMatch` / `LineReplace`) that avoid allocating a string per line. Use an explicitly typed lambda parameter to select them, e.g. `ScrubLines((CharSpan line) => ...)`; untyped lambdas bind the string overloads. + Engine semantics: * **Quarantine**: text produced by a replacement is never re-examined by other engine scrubbers. Legacy scrubbers run afterwards and can still modify it. diff --git a/docs/scrubbers.md b/docs/scrubbers.md index b15f944ad6..69a093d7db 100644 --- a/docs/scrubbers.md +++ b/docs/scrubbers.md @@ -26,6 +26,8 @@ A `Scrubber` is created via static factory methods and registered via `AddScrubb * `Scrubber.Match(matcher, minLength, maxLength)`: custom search logic. The matcher locates the next match within a segment. * `Scrubber.RemoveLinesContaining(...)`, `Scrubber.RemoveLines(...)`, `Scrubber.ReplaceLines(...)`, `Scrubber.RemoveEmptyLines()`: line scoped scrubbers. +`ScrubLines` and `ScrubLinesWithReplace` (and the corresponding `Scrubber` factories) also accept span based delegates (`LineMatch` / `LineReplace`) that avoid allocating a string per line. Use an explicitly typed lambda parameter to select them, e.g. `ScrubLines((CharSpan line) => ...)`; untyped lambdas bind the string overloads. + Engine semantics: * **Quarantine**: text produced by a replacement is never re-examined by other engine scrubbers. Legacy scrubbers run afterwards and can still modify it. diff --git a/src/ApplyScrubbersTests/LinePhaseTests.cs b/src/ApplyScrubbersTests/LinePhaseTests.cs index df18513734..df5bbe1fcc 100644 --- a/src/ApplyScrubbersTests/LinePhaseTests.cs +++ b/src/ApplyScrubbersTests/LinePhaseTests.cs @@ -111,6 +111,42 @@ public void SpanPredicateOverload() Assert.Equal("keep", result); } + [Fact] + public void UntypedLambda_BindsStringOverload() + { + // _.Contains('D') is valid for both string and span, so overload + // resolution priority must pick the string overload instead of + // reporting an ambiguity + var result = EngineRunner.Run( + "a\nD\nb", + Scrubber.RemoveLines(_ => _.Contains('D'))); + Assert.Equal("a\nb", result); + } + + [Fact] + public void SpanSugarOverloads() + { + var settings = new VerifySettings(); + // Explicitly typed span lambdas select the LineMatch/LineReplace overloads + settings.ScrubLines((CharSpan line) => line.StartsWith("remove".AsSpan())); + settings.ScrubLinesWithReplace((CharSpan line) => + { + if (line.StartsWith("replace".AsSpan())) + { + return LineResult.Replace("replaced"); + } + + return LineResult.Keep; + }); + // Untyped lambda still binds the string overload without ambiguity + settings.ScrubLines(_ => _.Contains("drop")); + + using var counter = Counter.Start(); + var builder = new StringBuilder("keep\nremove\nreplace me\ndrop me"); + ApplyScrubbers.ApplyForExtension("txt", builder, settings, counter); + Assert.Equal("keep\nreplaced", builder.ToString()); + } + [Fact] public void DropFirstMiddleLast() { diff --git a/src/Benchmarks/FilterLinesBenchmarks.cs b/src/Benchmarks/FilterLinesBenchmarks.cs index bdc20338ef..6d5e9160d7 100644 --- a/src/Benchmarks/FilterLinesBenchmarks.cs +++ b/src/Benchmarks/FilterLinesBenchmarks.cs @@ -11,6 +11,8 @@ public class FilterLinesBenchmarks EngineScrubberSet removeEvenSet = null!; EngineScrubberSet neverMatchesSet = null!; + EngineScrubberSet spanRemoveEvenSet = null!; + EngineScrubberSet spanNeverMatchesSet = null!; static Dictionary emptyContext = []; [GlobalSetup] @@ -27,6 +29,8 @@ public void Setup() removeEvenSet = EngineScrubberSet.ForScrubbers([Scrubber.RemoveLines(RemoveEvenLines)]); neverMatchesSet = EngineScrubberSet.ForScrubbers([Scrubber.RemoveLines(NeverMatches)]); + spanRemoveEvenSet = EngineScrubberSet.ForScrubbers([Scrubber.RemoveLines((LineMatch) RemoveEvenLinesSpan)]); + spanNeverMatchesSet = EngineScrubberSet.ForScrubbers([Scrubber.RemoveLines((LineMatch) NeverMatchesSpan)]); } static string CreateTestData(int lineCount, int charsPerLine) @@ -49,6 +53,12 @@ static bool RemoveEvenLines(string line) => // Never matches - no lines removed static bool NeverMatches(string line) => false; + // Span variants: no per-line string is materialized for these + static bool RemoveEvenLinesSpan(ReadOnlySpan line) => + line.Length > 0 && char.IsDigit(line[^1]) && (line[^1] - '0') % 2 == 0; + + static bool NeverMatchesSpan(ReadOnlySpan line) => false; + string Engine(EngineScrubberSet set, string content) { using var counter = Counter.Start(); @@ -114,4 +124,22 @@ public void Legacy_Large_NoMatches() [Benchmark] public string Engine_Large_NoMatches() => Engine(neverMatchesSet, largeInput); + + [Benchmark] + public string EngineSpan_Small() => Engine(spanRemoveEvenSet, smallInput); + + [Benchmark] + public string EngineSpan_Medium() => Engine(spanRemoveEvenSet, mediumInput); + + [Benchmark] + public string EngineSpan_Large() => Engine(spanRemoveEvenSet, largeInput); + + [Benchmark] + public string EngineSpan_Small_NoMatches() => Engine(spanNeverMatchesSet, smallInput); + + [Benchmark] + public string EngineSpan_Medium_NoMatches() => Engine(spanNeverMatchesSet, mediumInput); + + [Benchmark] + public string EngineSpan_Large_NoMatches() => Engine(spanNeverMatchesSet, largeInput); } diff --git a/src/Verify/Serialization/Scrubbers/Scrubber.cs b/src/Verify/Serialization/Scrubbers/Scrubber.cs index 8d19100933..cc0dadafd3 100644 --- a/src/Verify/Serialization/Scrubbers/Scrubber.cs +++ b/src/Verify/Serialization/Scrubbers/Scrubber.cs @@ -218,7 +218,11 @@ public static Scrubber RemoveLinesContaining(params string[] needles) => /// /// Remove any lines matching . + /// No per line string is allocated for span predicates. + /// Use an explicitly typed lambda parameter to select this overload, + /// e.g. RemoveLines((CharSpan line) => ...). /// + [OverloadResolutionPriority(-1)] public static Scrubber RemoveLines(LineMatch shouldRemove) { Ensure.NotNull(shouldRemove); @@ -236,7 +240,11 @@ public static Scrubber RemoveLines(Func shouldRemove) /// /// Process each line via . + /// No per line string is allocated for span based replacers. + /// Use an explicitly typed lambda parameter to select this overload, + /// e.g. ReplaceLines((CharSpan line) => ...). /// + [OverloadResolutionPriority(-1)] public static Scrubber ReplaceLines(LineReplace replace) { Ensure.NotNull(replace); diff --git a/src/Verify/Serialization/Scrubbers/VerifierSettings_ExtensionMappedGlobalScrubbers.cs b/src/Verify/Serialization/Scrubbers/VerifierSettings_ExtensionMappedGlobalScrubbers.cs index ebc019372c..9f9aff0578 100644 --- a/src/Verify/Serialization/Scrubbers/VerifierSettings_ExtensionMappedGlobalScrubbers.cs +++ b/src/Verify/Serialization/Scrubbers/VerifierSettings_ExtensionMappedGlobalScrubbers.cs @@ -79,6 +79,16 @@ public static void ScrubLinesContaining(string extension, StringComparison compa public static void ScrubLines(string extension, Func removeLine) => AddScrubber(extension, Scrubber.RemoveLines(removeLine)); + /// + /// Remove any lines matching from the test results. + /// No per line string is allocated for span predicates. + /// Use an explicitly typed lambda parameter to select this overload, + /// e.g. ScrubLines(extension, (CharSpan line) => ...). + /// + [OverloadResolutionPriority(-1)] + public static void ScrubLines(string extension, LineMatch removeLine) => + AddScrubber(extension, Scrubber.RemoveLines(removeLine)); + /// /// Remove any lines matching from the test results. /// @@ -119,6 +129,16 @@ public static void ScrubInlineGuids(string extension, ScrubberLocation location) public static void ScrubLinesWithReplace(string extension, Func replaceLine) => AddScrubber(extension, Scrubber.ReplaceLines(replaceLine)); + /// + /// Scrub lines with an optional replace. + /// No per line string is allocated for span based replacers. + /// Use an explicitly typed lambda parameter to select this overload, + /// e.g. ScrubLinesWithReplace(extension, (CharSpan line) => ...). + /// + [OverloadResolutionPriority(-1)] + public static void ScrubLinesWithReplace(string extension, LineReplace replaceLine) => + AddScrubber(extension, Scrubber.ReplaceLines(replaceLine)); + /// /// Scrub lines with an optional replace. /// can return the input to ignore the line, or return a different string to replace it. diff --git a/src/Verify/Serialization/Scrubbers/VerifierSettings_GlobalScrubbers.cs b/src/Verify/Serialization/Scrubbers/VerifierSettings_GlobalScrubbers.cs index f4b0ed5c8b..0df8925f6d 100644 --- a/src/Verify/Serialization/Scrubbers/VerifierSettings_GlobalScrubbers.cs +++ b/src/Verify/Serialization/Scrubbers/VerifierSettings_GlobalScrubbers.cs @@ -67,6 +67,16 @@ public static void ScrubLinesContaining(StringComparison comparison, ScrubberLoc public static void ScrubLines(Func removeLine) => AddScrubber(Scrubber.RemoveLines(removeLine)); + /// + /// Remove any lines matching from the test results. + /// No per line string is allocated for span predicates. + /// Use an explicitly typed lambda parameter to select this overload, + /// e.g. ScrubLines((CharSpan line) => ...). + /// + [OverloadResolutionPriority(-1)] + public static void ScrubLines(LineMatch removeLine) => + AddScrubber(Scrubber.RemoveLines(removeLine)); + /// /// Remove any lines matching from the test results. /// @@ -188,6 +198,16 @@ public static void ScrubInlineGuids(ScrubberLocation location) => public static void ScrubLinesWithReplace(Func replaceLine) => AddScrubber(Scrubber.ReplaceLines(replaceLine)); + /// + /// Scrub lines with an optional replace. + /// No per line string is allocated for span based replacers. + /// Use an explicitly typed lambda parameter to select this overload, + /// e.g. ScrubLinesWithReplace((CharSpan line) => ...). + /// + [OverloadResolutionPriority(-1)] + public static void ScrubLinesWithReplace(LineReplace replaceLine) => + AddScrubber(Scrubber.ReplaceLines(replaceLine)); + /// /// Scrub lines with an optional replace. /// can return the input to ignore the line, or return a different string to replace it. diff --git a/src/Verify/Serialization/Scrubbers/VerifySettings_ExtensionMappedInstanceScrubbers.cs b/src/Verify/Serialization/Scrubbers/VerifySettings_ExtensionMappedInstanceScrubbers.cs index 6c5e69764f..ac4f834b90 100644 --- a/src/Verify/Serialization/Scrubbers/VerifySettings_ExtensionMappedInstanceScrubbers.cs +++ b/src/Verify/Serialization/Scrubbers/VerifySettings_ExtensionMappedInstanceScrubbers.cs @@ -116,6 +116,16 @@ public void ScrubInlineGuids(string extension, ScrubberLocation location) => public void ScrubLines(string extension, Func removeLine) => AddScrubber(extension, Scrubber.RemoveLines(removeLine)); + /// + /// Remove any lines matching from the test results. + /// No per line string is allocated for span predicates. + /// Use an explicitly typed lambda parameter to select this overload, + /// e.g. ScrubLines(extension, (CharSpan line) => ...). + /// + [OverloadResolutionPriority(-1)] + public void ScrubLines(string extension, LineMatch removeLine) => + AddScrubber(extension, Scrubber.RemoveLines(removeLine)); + /// /// Remove any lines matching from the test results. /// @@ -130,6 +140,16 @@ public void ScrubLines(string extension, Func removeLine, Scrubber public void ScrubLinesWithReplace(string extension, Func replaceLine) => AddScrubber(extension, Scrubber.ReplaceLines(replaceLine)); + /// + /// Scrub lines with an optional replace. + /// No per line string is allocated for span based replacers. + /// Use an explicitly typed lambda parameter to select this overload, + /// e.g. ScrubLinesWithReplace(extension, (CharSpan line) => ...). + /// + [OverloadResolutionPriority(-1)] + public void ScrubLinesWithReplace(string extension, LineReplace replaceLine) => + AddScrubber(extension, Scrubber.ReplaceLines(replaceLine)); + /// /// Scrub lines with an optional replace. /// can return the input to ignore the line, or return a different string to replace it. diff --git a/src/Verify/Serialization/Scrubbers/VerifySettings_InstanceScrubbers.cs b/src/Verify/Serialization/Scrubbers/VerifySettings_InstanceScrubbers.cs index 06d4b8c528..da5a089d4a 100644 --- a/src/Verify/Serialization/Scrubbers/VerifySettings_InstanceScrubbers.cs +++ b/src/Verify/Serialization/Scrubbers/VerifySettings_InstanceScrubbers.cs @@ -219,6 +219,16 @@ public void ScrubInlineDates( public void ScrubLines(Func removeLine) => AddScrubber(Scrubber.RemoveLines(removeLine)); + /// + /// Remove any lines matching from the test results. + /// No per line string is allocated for span predicates. + /// Use an explicitly typed lambda parameter to select this overload, + /// e.g. ScrubLines((CharSpan line) => ...). + /// + [OverloadResolutionPriority(-1)] + public void ScrubLines(LineMatch removeLine) => + AddScrubber(Scrubber.RemoveLines(removeLine)); + /// /// Remove any lines matching from the test results. /// @@ -233,6 +243,16 @@ public void ScrubLines(Func removeLine, ScrubberLocation location) public void ScrubLinesWithReplace(Func replaceLine) => AddScrubber(Scrubber.ReplaceLines(replaceLine)); + /// + /// Scrub lines with an optional replace. + /// No per line string is allocated for span based replacers. + /// Use an explicitly typed lambda parameter to select this overload, + /// e.g. ScrubLinesWithReplace((CharSpan line) => ...). + /// + [OverloadResolutionPriority(-1)] + public void ScrubLinesWithReplace(LineReplace replaceLine) => + AddScrubber(Scrubber.ReplaceLines(replaceLine)); + /// /// Scrub lines with an optional replace. /// can return the input to ignore the line, or return a different string to replace it. diff --git a/src/Verify/SettingsTask_Scrubbing.cs b/src/Verify/SettingsTask_Scrubbing.cs index b9edc406a4..3558e24970 100644 --- a/src/Verify/SettingsTask_Scrubbing.cs +++ b/src/Verify/SettingsTask_Scrubbing.cs @@ -257,6 +257,15 @@ public SettingsTask ScrubLines(Func removeLine) return this; } + /// + [OverloadResolutionPriority(-1)] + [Pure] + public SettingsTask ScrubLines(LineMatch removeLine) + { + CurrentSettings.ScrubLines(removeLine); + return this; + } + /// [Obsolete(locationObsolete)] [Pure] @@ -271,6 +280,15 @@ public SettingsTask ScrubLines(string extension, Func removeLine) return this; } + /// + [OverloadResolutionPriority(-1)] + [Pure] + public SettingsTask ScrubLines(string extension, LineMatch removeLine) + { + CurrentSettings.ScrubLines(extension, removeLine); + return this; + } + /// [Obsolete(locationObsolete)] [Pure] @@ -285,6 +303,15 @@ public SettingsTask ScrubLinesWithReplace(Func replaceLine) return this; } + /// + [OverloadResolutionPriority(-1)] + [Pure] + public SettingsTask ScrubLinesWithReplace(LineReplace replaceLine) + { + CurrentSettings.ScrubLinesWithReplace(replaceLine); + return this; + } + /// [Obsolete(locationObsolete)] [Pure] @@ -299,6 +326,15 @@ public SettingsTask ScrubLinesWithReplace(string extension, Func + [OverloadResolutionPriority(-1)] + [Pure] + public SettingsTask ScrubLinesWithReplace(string extension, LineReplace replaceLine) + { + CurrentSettings.ScrubLinesWithReplace(extension, replaceLine); + return this; + } + /// [Obsolete(locationObsolete)] [Pure] diff --git a/src/VerifyCore.slnf b/src/VerifyCore.slnf index e521ab1c5f..859263d304 100644 --- a/src/VerifyCore.slnf +++ b/src/VerifyCore.slnf @@ -2,6 +2,7 @@ "solution": { "path": "Verify.slnx", "projects": [ + "ApplyScrubbersTests\\ApplyScrubbersTests.csproj", "DisableScrubbersTests\\DisableScrubbersTests.csproj", "Benchmarks\\Benchmarks.csproj", "RawTempUsage\\RawTempUsage.csproj", From 4a75188b7e131b70b2c6f500fad981e8e6417a94 Mon Sep 17 00:00:00 2001 From: Simon Cropp Date: Sat, 11 Jul 2026 14:56:59 +1000 Subject: [PATCH 03/33] . --- src/Verify.Tests/DateScrubberTests.cs | 25 +++++++++++++++++++ .../Serialization/Scrubbers/DateMatchers.cs | 10 +++++--- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/src/Verify.Tests/DateScrubberTests.cs b/src/Verify.Tests/DateScrubberTests.cs index 2251ccb28b..33f3449b4e 100644 --- a/src/Verify.Tests/DateScrubberTests.cs +++ b/src/Verify.Tests/DateScrubberTests.cs @@ -90,6 +90,31 @@ await Verify(result) .UseTextForParameters(name); } + [Fact] + public void StandardFormat_AllCultures() + { + // Single char standard formats expand per culture; the digit prefilter + // analysis runs on the expanded pattern and must never suppress a real match + foreach (var culture in CultureInfo.GetCultures(CultureTypes.AllCultures)) + { + using var counter = Counter.Start(); + var dateTime = DateTime.Now; + var value = dateTime.ToString("d", culture); + var result = Scrub($"a {value} b", DateMatchers.DateTimes("d", culture), counter); + if (result == "a DateTime_1 b") + { + continue; + } + + throw new( + $""" + {culture.DisplayName} {culture.Name} + {result} + {value} + """); + } + } + [Fact] public void ReplaceDateTimes_AllCultures() { diff --git a/src/Verify/Serialization/Scrubbers/DateMatchers.cs b/src/Verify/Serialization/Scrubbers/DateMatchers.cs index 616a8deefe..c699b838a6 100644 --- a/src/Verify/Serialization/Scrubbers/DateMatchers.cs +++ b/src/Verify/Serialization/Scrubbers/DateMatchers.cs @@ -126,7 +126,9 @@ static Scrubber[] BuildForFormats(string format, Culture culture, Func Date: Sat, 11 Jul 2026 17:02:21 +1000 Subject: [PATCH 04/33] . --- .../Serialization/Scrubbers/DateMatchers.cs | 41 ++++++---- .../Scrubbers/DirectoryReplacements.cs | 47 ++++++++++- .../Scrubbers/EngineScrubberSet.cs | 5 +- .../Serialization/Scrubbers/GuidMatcher.cs | 14 +++- .../Serialization/Scrubbers/ScrubEngine.cs | 77 ++++++++++++++++++- .../Serialization/Scrubbers/Scrubber.cs | 32 +++++++- .../Serialization/Scrubbers/ScrubberKind.cs | 10 +++ .../Scrubbers/SharedScrubber_Dates.cs | 15 +++- .../Scrubbers/SharedScrubber_Guids.cs | 6 +- .../VerifySettings_InstanceScrubbers.cs | 5 ++ 10 files changed, 225 insertions(+), 27 deletions(-) diff --git a/src/Verify/Serialization/Scrubbers/DateMatchers.cs b/src/Verify/Serialization/Scrubbers/DateMatchers.cs index c699b838a6..04864fa5ac 100644 --- a/src/Verify/Serialization/Scrubbers/DateMatchers.cs +++ b/src/Verify/Serialization/Scrubbers/DateMatchers.cs @@ -129,24 +129,35 @@ static Scrubber Single(string format, Culture culture, ParseWindow parse) // Single char standard formats expand to the culture's pattern, so the // prefilter analysis runs on what will actually render var digitPrefilter = HasDigitPrefilter(culture.DateTimeFormat.ExpandFormat(format)); - return Scrubber.Window( - Math.Max(1, min), - max, - (window, counter, _) => + + string? Match(CharSpan window, Counter counter, IReadOnlyDictionary context) + { + if (!counter.ScrubDateTimes) { - if (!counter.ScrubDateTimes) - { - return null; - } + return null; + } - if (digitPrefilter && - !char.IsDigit(window[0])) - { - return null; - } + return parse(window, counter); + } + + if (digitPrefilter) + { + // The engine anchor jumps between digits, so no-match text is scanned + // vectorized instead of per position + return Scrubber.AnchoredWindow( + Math.Max(1, min), + max, + Match, + requireWordBoundary: false, + anchor: WindowAnchor.Digit, + anchorChar: default, + anchorOffset: 0); + } - return parse(window, counter); - }); + return Scrubber.Window( + Math.Max(1, min), + max, + Match); } // True when the (expanded) format is guaranteed to render a digit first, diff --git a/src/Verify/Serialization/Scrubbers/DirectoryReplacements.cs b/src/Verify/Serialization/Scrubbers/DirectoryReplacements.cs index 8dafa9f204..b99962f39e 100644 --- a/src/Verify/Serialization/Scrubbers/DirectoryReplacements.cs +++ b/src/Verify/Serialization/Scrubbers/DirectoryReplacements.cs @@ -22,6 +22,7 @@ public Pair(string find, string replace) static List items = []; static int shortestFindLength = int.MaxValue; + static string itemAnchors = ""; // Length of the shortest Find, so callers can cheaply skip values that are // too short to contain any replacement. int.MaxValue when there are none. @@ -29,12 +30,41 @@ public Pair(string find, string replace) internal static List Items => items; + // The distinct first chars of the Finds, so scans can skip to candidate + // positions with a vectorized IndexOfAny instead of probing every position + internal static string ItemAnchors => itemAnchors; + + internal static string BuildAnchors(List pairs) + { + var anchors = new List(); + foreach (var pair in pairs) + { + var first = pair.Find[0]; + if (!anchors.Contains(first)) + { + anchors.Add(first); + } + + // '/' and '\' are equivalent during matching + if (first == '/' && + !anchors.Contains('\\')) + { + anchors.Add('\\'); + } + } + + return new([.. anchors]); + } + public static void Replace(StringBuilder builder) => - Replace(builder, items); + Replace(builder, items, itemAnchors); + + public static void Replace(StringBuilder builder, List pairs) => + Replace(builder, pairs, BuildAnchors(pairs)); // Legacy pass entry point: materialize once, scan with the span matcher, apply // matches position-descending so earlier indexes stay valid. - public static void Replace(StringBuilder builder, List pairs) + static void Replace(StringBuilder builder, List pairs, string anchors) { #if DEBUG var finds = pairs.Select(_ => _.Find).ToList(); @@ -66,6 +96,18 @@ public static void Replace(StringBuilder builder, List pairs) List<(int Index, int Length, string Value)>? matches = null; for (var position = 0; position + shortest <= span.Length; position++) { + var skip = span[position..].IndexOfAny(anchors.AsSpan()); + if (skip < 0) + { + break; + } + + position += skip; + if (position + shortest > span.Length) + { + break; + } + if (!TryMatchAt(span, position, null, null, pairs, out var matchLength, out var replacement)) { continue; @@ -128,6 +170,7 @@ public static void UseAssembly(string? solutionDir, string projectDir) .OrderByDescending(_ => _.Find.Length) .ToList(); shortestFindLength = items.Count == 0 ? int.MaxValue : items[^1].Find.Length; + itemAnchors = BuildAnchors(items); } static void AddProjectAndSolutionReplacements(string? solutionDir, string projectDir, List replacements) diff --git a/src/Verify/Serialization/Scrubbers/EngineScrubberSet.cs b/src/Verify/Serialization/Scrubbers/EngineScrubberSet.cs index ba30ccad4b..7d8650594f 100644 --- a/src/Verify/Serialization/Scrubbers/EngineScrubberSet.cs +++ b/src/Verify/Serialization/Scrubbers/EngineScrubberSet.cs @@ -102,7 +102,8 @@ public static EngineScrubberSet ForExtension(VerifySettings settings, string ext } // Property-value pass: instance and global only. Extension-mapped scrubbers - // are excluded, matching the legacy pipeline. + // are excluded, matching the legacy pipeline. The merged set is cached on the + // settings instance since this runs once per serialized string value. public static EngineScrubberSet ForPropertyValue(VerifySettings settings) { var instance = settings.InstanceSpanScrubbers; @@ -111,7 +112,7 @@ public static EngineScrubberSet ForPropertyValue(VerifySettings settings) return globalOnlyCache ??= Build(null, null, null, VerifierSettings.GlobalSpanScrubbers); } - return Build(instance, null, null, VerifierSettings.GlobalSpanScrubbers); + return settings.PropertyValueSetCache ??= Build(instance, null, null, VerifierSettings.GlobalSpanScrubbers); } static bool IsNullOrEmpty(List? list) => diff --git a/src/Verify/Serialization/Scrubbers/GuidMatcher.cs b/src/Verify/Serialization/Scrubbers/GuidMatcher.cs index de8a6efa92..33f6b189f0 100644 --- a/src/Verify/Serialization/Scrubbers/GuidMatcher.cs +++ b/src/Verify/Serialization/Scrubbers/GuidMatcher.cs @@ -1,8 +1,16 @@ -// The inline guid scrubber: a fixed 36 char window over the canonical "D" format, -// with a dash prefilter so the parse is only attempted at plausible positions. +// The inline guid scrubber: a fixed 36 char window over the canonical "D" format. +// The engine anchor jumps between '-' chars at offset 8 (the first dash of the "D" +// format), so no-match text is scanned vectorized instead of per position. static class GuidMatcher { - public static readonly Scrubber Instance = Scrubber.Window(36, 36, Match, requireWordBoundary: true); + public static readonly Scrubber Instance = Scrubber.AnchoredWindow( + 36, + 36, + Match, + requireWordBoundary: true, + anchor: WindowAnchor.Char, + anchorChar: '-', + anchorOffset: 8); static string? Match(CharSpan window, Counter counter, IReadOnlyDictionary context) { diff --git a/src/Verify/Serialization/Scrubbers/ScrubEngine.cs b/src/Verify/Serialization/Scrubbers/ScrubEngine.cs index 4b7aaf88cc..63d69b3bd4 100644 --- a/src/Verify/Serialization/Scrubbers/ScrubEngine.cs +++ b/src/Verify/Serialization/Scrubbers/ScrubEngine.cs @@ -36,6 +36,7 @@ public static string Run( return AssembleString(chunks); } + // target must already contain source, so an unchanged result leaves it untouched public static void RunToBuilder( string source, EngineScrubberSet set, @@ -46,13 +47,20 @@ public static void RunToBuilder( { var working = Scrubber.NormalizeNewlines(source); var chunks = ScrubCore(working, set, counter, context, applyDirectoryReplacements); - target.Clear(); if (chunks == null) { + if (ReferenceEquals(working, source)) + { + return; + } + + target.Clear(); target.Append(working); return; } + target.Clear(); + var total = 0; foreach (var chunk in chunks) { @@ -155,10 +163,24 @@ static bool TryFindDirectoryMatch( out string replacement) { var span = chunks[chunkIndex].Span; + var anchors = DirectoryReplacements.ItemAnchors.AsSpan(); var beforeSegment = chunkIndex > 0 ? chunks[chunkIndex - 1].Last : (char?) null; var afterSegment = chunkIndex + 1 < chunks.Count ? chunks[chunkIndex + 1].First : (char?) null; for (var position = 0; position + shortest <= span.Length; position++) { + // Skip to the next candidate first char + var skip = span[position..].IndexOfAny(anchors); + if (skip < 0) + { + break; + } + + position += skip; + if (position + shortest > span.Length) + { + break; + } + if (DirectoryReplacements.TryMatchAt(span, position, beforeSegment, afterSegment, pairs, out matchLength, out replacement)) { matchStart = position; @@ -375,6 +397,17 @@ static bool TryFindWindowMatch( for (var position = regionStart; position <= regionEnd - min; position++) { + if (scrubber.Anchor != WindowAnchor.None) + { + // A match can only start where the anchor appears at the fixed + // offset, so jump to the next candidate + position = NextAnchoredPosition(span, position, regionEnd, scrubber); + if (position > regionEnd - min) + { + break; + } + } + if (scrubber.RequireWordBoundary && PreviousChar(chunks, chunkIndex, position) is { } before && char.IsLetterOrDigit(before)) @@ -419,6 +452,48 @@ static bool TryFindWindowMatch( return false; } + // The next position at or after the given one where the scrubber's anchor + // appears at its offset from the window start. Returns past regionEnd when no + // candidate remains. + static int NextAnchoredPosition(CharSpan span, int position, int regionEnd, Scrubber scrubber) + { + var searchFrom = position + scrubber.AnchorOffset; + if (searchFrom >= regionEnd) + { + return regionEnd; + } + + var region = span[searchFrom..regionEnd]; + int found; + if (scrubber.Anchor == WindowAnchor.Char) + { + found = region.IndexOf(scrubber.AnchorChar); + } + else + { +#if NET8_0_OR_GREATER + found = region.IndexOfAnyInRange('0', '9'); +#else + found = -1; + for (var index = 0; index < region.Length; index++) + { + if (char.IsDigit(region[index])) + { + found = index; + break; + } + } +#endif + } + + if (found < 0) + { + return regionEnd; + } + + return searchFrom + found - scrubber.AnchorOffset; + } + static bool TryFindCustomMatch( List chunks, int chunkIndex, diff --git a/src/Verify/Serialization/Scrubbers/Scrubber.cs b/src/Verify/Serialization/Scrubbers/Scrubber.cs index cc0dadafd3..b2595773cf 100644 --- a/src/Verify/Serialization/Scrubbers/Scrubber.cs +++ b/src/Verify/Serialization/Scrubbers/Scrubber.cs @@ -26,6 +26,9 @@ public sealed class Scrubber internal int? MaxLength { get; } internal StringComparison Comparison { get; } internal bool RequireWordBoundary { get; } + internal WindowAnchor Anchor { get; } + internal char AnchorChar { get; } + internal int AnchorOffset { get; } internal (string Find, string Replacement)[]? Pairs { get; } internal string[]? Needles { get; } internal WindowMatch? WindowMatcher { get; } @@ -48,7 +51,10 @@ public sealed class Scrubber LineMatch? lineMatcher = null, Func? lineStringMatcher = null, LineReplace? lineReplacer = null, - Func? lineStringReplacer = null) + Func? lineStringReplacer = null, + WindowAnchor anchor = WindowAnchor.None, + char anchorChar = default, + int anchorOffset = 0) { Kind = kind; MinLength = minLength; @@ -63,6 +69,9 @@ public sealed class Scrubber LineStringMatcher = lineStringMatcher; LineReplacer = lineReplacer; LineStringReplacer = lineStringReplacer; + Anchor = anchor; + AnchorChar = anchorChar; + AnchorOffset = anchorOffset; } internal bool IsLineDrop => @@ -159,6 +168,27 @@ public static Scrubber Window( windowMatcher: matcher); } + // A Window scrubber whose matches can only start where the anchor appears at + // anchorOffset from the window start. Used by the built-in guid and date + // scrubbers so no-match scans skip between candidate positions. + internal static Scrubber AnchoredWindow( + int minLength, + int maxLength, + WindowMatch matcher, + bool requireWordBoundary, + WindowAnchor anchor, + char anchorChar, + int anchorOffset) => + new( + ScrubberKind.Window, + minLength: minLength, + maxLength: maxLength, + requireWordBoundary: requireWordBoundary, + windowMatcher: matcher, + anchor: anchor, + anchorChar: anchorChar, + anchorOffset: anchorOffset); + /// /// Find matches using custom search logic. /// : segments shorter than this are skipped (null scans everything). diff --git a/src/Verify/Serialization/Scrubbers/ScrubberKind.cs b/src/Verify/Serialization/Scrubbers/ScrubberKind.cs index 63923eb2af..41634e2696 100644 --- a/src/Verify/Serialization/Scrubbers/ScrubberKind.cs +++ b/src/Verify/Serialization/Scrubbers/ScrubberKind.cs @@ -10,3 +10,13 @@ enum ScrubberKind LineTransformSpan, LineTransformString, } + +// A cheap vectorized scan target for Window scrubbers: a match can only start +// where the anchor appears at the given offset from the window start, so the +// engine can jump between candidate positions instead of probing every one. +enum WindowAnchor +{ + None, + Char, + Digit, +} diff --git a/src/Verify/Serialization/Scrubbers/SharedScrubber_Dates.cs b/src/Verify/Serialization/Scrubbers/SharedScrubber_Dates.cs index 733f5bb506..10e3ad0e54 100644 --- a/src/Verify/Serialization/Scrubbers/SharedScrubber_Dates.cs +++ b/src/Verify/Serialization/Scrubbers/SharedScrubber_Dates.cs @@ -178,7 +178,8 @@ public bool TryConvertDateTime(CharSpan value, [NotNullWhen(true)] out string? r { if (ScrubDateTimes) { - if (TryParseDateTime(value, "yyyy-MM-ddTHH:mm:ss.FFFFFFFK", out var date)) + if (CanBeIsoDate(value) && + TryParseDateTime(value, "yyyy-MM-ddTHH:mm:ss.FFFFFFFK", out var date)) { result = Convert(date); return true; @@ -198,6 +199,15 @@ public bool TryConvertDateTime(CharSpan value, [NotNullWhen(true)] out string? r return false; } + // The ISO format is "yyyy-MM-ddTHH:mm:ss.FFFFFFFK": at least 19 chars, starting + // with a 4 digit year then '-'. TryParseExact with DateTimeStyles.None allows no + // whitespace, so values failing this shape can never parse and the attempt can + // be skipped. + static bool CanBeIsoDate(CharSpan value) => + value.Length >= 19 && + char.IsDigit(value[0]) && + value[4] == '-'; + static bool TryParseDateTime(CharSpan value, string format, out DateTime dateTime) => DateTime.TryParseExact(value, format, null, DateTimeStyles.None, out dateTime); @@ -205,7 +215,8 @@ public bool TryConvertDateTimeOffset(CharSpan value, [NotNullWhen(true)] out str { if (ScrubDateTimes) { - if (TryParseDateTimeOffset(value, "yyyy-MM-ddTHH:mm:ss.FFFFFFFK", out var date)) + if (CanBeIsoDate(value) && + TryParseDateTimeOffset(value, "yyyy-MM-ddTHH:mm:ss.FFFFFFFK", out var date)) { result = Convert(date); return true; diff --git a/src/Verify/Serialization/Scrubbers/SharedScrubber_Guids.cs b/src/Verify/Serialization/Scrubbers/SharedScrubber_Guids.cs index 1dbeeaa6ac..038c7b107d 100644 --- a/src/Verify/Serialization/Scrubbers/SharedScrubber_Guids.cs +++ b/src/Verify/Serialization/Scrubbers/SharedScrubber_Guids.cs @@ -28,7 +28,11 @@ public bool TryConvertGuid(CharSpan value, [NotNullWhen(true)] out string? resul { if (ScrubGuids) { - if (Guid.TryParse(value, out var guid)) + // The shortest parseable guid (the "N" format) is 32 chars. Whitespace + // padding, which Guid.TryParse trims, only adds length, so anything + // shorter can never parse and the attempt can be skipped. + if (value.Length >= 32 && + Guid.TryParse(value, out var guid)) { result = Convert(guid); return true; diff --git a/src/Verify/Serialization/Scrubbers/VerifySettings_InstanceScrubbers.cs b/src/Verify/Serialization/Scrubbers/VerifySettings_InstanceScrubbers.cs index da5a089d4a..4c2505ed57 100644 --- a/src/Verify/Serialization/Scrubbers/VerifySettings_InstanceScrubbers.cs +++ b/src/Verify/Serialization/Scrubbers/VerifySettings_InstanceScrubbers.cs @@ -6,6 +6,10 @@ public partial class VerifySettings internal List? InstanceSpanScrubbers; + // Cached merged set for the property value path, which runs once per + // serialized string value. Invalidated on registration. + internal EngineScrubberSet? PropertyValueSetCache; + const string locationObsolete = "ScrubberLocation is ignored; span scrubber ordering is engine determined. Use the overload without ScrubberLocation."; /// @@ -16,6 +20,7 @@ public void AddScrubber(Scrubber scrubber) Ensure.NotNull(scrubber); InstanceSpanScrubbers ??= []; InstanceSpanScrubbers.Add(scrubber); + PropertyValueSetCache = null; } internal bool ScrubbersEnabled { get; private set; } = true; From 9784975bb90456ffb120d77c6b0e101c5cc5b3ad Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Sat, 11 Jul 2026 07:04:28 +0000 Subject: [PATCH 05/33] Docs changes --- docs/dates.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/dates.md b/docs/dates.md index a3b17e3c59..cf7cdb72bc 100644 --- a/docs/dates.md +++ b/docs/dates.md @@ -308,7 +308,7 @@ public Task InferredNamedDateFluent() .AddNamedDate(namedDate); } ``` -snippet source | anchor +snippet source | anchor Result: From 1840c976b83e1c1b04bf28955694287d224b37f9 Mon Sep 17 00:00:00 2001 From: Simon Cropp Date: Wed, 15 Jul 2026 07:32:30 +1000 Subject: [PATCH 06/33] refs or cleanup --- global.json | 2 +- src/Directory.Packages.props | 20 ++++++++++---------- usages/global.json | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/global.json b/global.json index e4b3404556..1485c54fe8 100644 --- a/global.json +++ b/global.json @@ -1,6 +1,6 @@ { "sdk": { - "version": "11.0.100-preview.5.26302.115", + "version": "11.0.100-preview.6.26359.118", "allowPrerelease": true, "rollForward": "latestFeature" }, diff --git a/src/Directory.Packages.props b/src/Directory.Packages.props index 67375ccdd0..9606c8119b 100644 --- a/src/Directory.Packages.props +++ b/src/Directory.Packages.props @@ -4,8 +4,8 @@ true - - + + @@ -13,27 +13,27 @@ - + - - + + - - + + - + - - + + diff --git a/usages/global.json b/usages/global.json index 5ca0104fb0..8cf7637726 100644 --- a/usages/global.json +++ b/usages/global.json @@ -1,6 +1,6 @@ { "sdk": { - "version": "10.0.301", + "version": "10.0.302", "allowPrerelease": true, "rollForward": "latestFeature" } From 742a59fc8533aac51f8764d41f6aa87fa5788cef Mon Sep 17 00:00:00 2001 From: Simon Cropp Date: Sat, 18 Jul 2026 10:00:10 +1000 Subject: [PATCH 07/33] Update ScrubEngine_Lines.cs --- .../Scrubbers/ScrubEngine_Lines.cs | 42 ++++++++++--------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/src/Verify/Serialization/Scrubbers/ScrubEngine_Lines.cs b/src/Verify/Serialization/Scrubbers/ScrubEngine_Lines.cs index d3278d9225..1789a971ad 100644 --- a/src/Verify/Serialization/Scrubbers/ScrubEngine_Lines.cs +++ b/src/Verify/Serialization/Scrubbers/ScrubEngine_Lines.cs @@ -100,38 +100,40 @@ void FlushRun() return null; } - var chunks = new List(items.Count * 2); + var chunks = new List(items.Count + 1); for (var index = 0; index < items.Count; index++) { - if (index > 0) - { - chunks.Add(new(newlineText, 0, 1, scannable: true)); - } - var item = items[index]; - if (item.Length == 0) - { - continue; - } + var needsSeparator = index < items.Count - 1 || + (endsWithNewline && !set.TrimOuterEmptyLines); if (item.Fresh is { } fresh) { // Transformed line text is fresh source, scannable by inline scrubbers - chunks.Add(new(fresh, 0, fresh.Length, scannable: true)); + if (fresh.Length > 0) + { + chunks.Add(new(fresh, 0, fresh.Length, scannable: true)); + } + + if (needsSeparator) + { + chunks.Add(new(newlineText, 0, 1, scannable: true)); + } + + continue; } - else + + // A separator is only needed when another item follows (the run ended at + // a dropped or transformed line, so its last line was terminated) or when + // the document ends with a newline. Either way the char after the run in + // the source is '\n', so the separator folds into the slice. + var length = item.End - item.Start + (needsSeparator ? 1 : 0); + if (length > 0) { - chunks.Add(new(source, item.Start, item.End - item.Start, scannable: true)); + chunks.Add(new(source, item.Start, length, scannable: true)); } } - if (endsWithNewline && - items.Count > 0 && - !set.TrimOuterEmptyLines) - { - chunks.Add(new(newlineText, 0, 1, scannable: true)); - } - return chunks; } From 2d260fa326f71759d2ed2367dc9822ce2c48b8f4 Mon Sep 17 00:00:00 2001 From: Simon Cropp Date: Sat, 18 Jul 2026 16:12:38 +1000 Subject: [PATCH 08/33] Create release-notes-32.0.0.md --- release-notes-32.0.0.md | 125 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 release-notes-32.0.0.md diff --git a/release-notes-32.0.0.md b/release-notes-32.0.0.md new file mode 100644 index 0000000000..b4984717ff --- /dev/null +++ b/release-notes-32.0.0.md @@ -0,0 +1,125 @@ +# Verify 32.0.0 release notes + +## Summary + +This release replaces the internal text-scrubbing pipeline with a new **span based scrub engine** and +adds a public **`Scrubber`** API. The engine materializes each document once, tracks it as chunks, and +quarantines replacements so scrubbers no longer re-process each other's output. The result is large CPU +and allocation reductions for the built-in scrubbers, plus a first-class way to define custom scrubbers. + +It is a **major version bump** because, although the public API is source compatible (see below), the +change is **behaviorally breaking in some configurations**: scrubber execution order and interaction are +now engine determined, so a subset of setups will produce different `.verified.` output and require +re-accepting snapshots. + +The full behavior is documented in [scrubbers.md](/docs/scrubbers.md#the-scrub-engine). + +## Why + +The old pipeline was `StringBuilder` based. Every registered scrubber re-round-tripped the whole +document (`ToString()` + rebuild, or chunk-walking with cross-chunk carryover buffers), so cost scaled +with `scrubbers x document size` and allocated heavily even when nothing matched. The new engine does a +single pass with vectorized candidate scanning and returns the original string unchanged (zero copy) +when no scrubber fired. + +## Performance + +Measured on a 31 KB document (p99 of a scan of 33,707 real `.verified.` files), AMD Ryzen 9 5900X, +.NET 10, `MemoryDiagnoser`. "Old" runs the pre-engine implementations over identical input. + +| Scenario | Old | New | Speedup | Allocation | +|---|--:|--:|--:|--:| +| Inline guid scan, no match | 105.5 us / 61.8 KB | 6.6 us / 1.2 KB | 16x | 51x less | +| Inline DateTime (ISO) scan, no match | 709.9 us / 122.4 KB | 72.7 us / 1.2 KB | 9.8x | 100x less | +| Inline DateTimeOffset, typical | 69.7 us / 15.5 KB | 4.1 us / 10.8 KB | 17x | — | +| `ScrubLinesContaining`, 508 lines | 16.2 us / 186 KB | 10.3 us / 59 KB | 1.6x | 3.2x less | +| Whole pipeline (guids + dates + line drops, one pass) | 884.0 us / 359 KB | 145.1 us / 103 KB | 6.1x | 3.5x less | +| Serialization string probe (per value, all misses) | 144 ns | 42 ns | 3.4x | — | + +The one scenario that regresses is predicate line removal (`ScrubLines(Func)`) on small and +medium documents (~0.5 us at p50, ~10 us at 1,000 lines) — the cost of the composable pipeline. It wins +at scale (5x at 10,000 lines) and is avoidable via the new span predicate overload or +`ScrubLinesContaining`. + +## Breaking changes + +### API (source compatible, with obsoletions) + +No public API was removed and no signatures changed incompatibly. However, the `ScrubberLocation` +parameter is now meaningless for the built-in scrub methods (ordering is engine determined), so those +overloads are marked `[Obsolete]`: + +`ScrubLinesContaining`, `ScrubLines`, `ScrubLinesWithReplace`, `ScrubEmptyLines`, `ScrubInlineGuids`, +`ScrubInlineDates`, `ScrubInlineDateTimes`, `ScrubInlineDateTimeOffsets`, `ScrubMachineName`, +`ScrubUserName` (at global, instance, extension mapped, and fluent levels). + +Existing calls still compile, but a call that passes a `ScrubberLocation` produces an obsolete warning. +Projects using `TreatWarningsAsErrors` will need to drop the `ScrubberLocation` argument. The +`ScrubberLocation` on the low-level `AddScrubber(Action, ...)` overloads is **unchanged** +and still honored. + +### Behavior (snapshot output) + +Even source-identical code can now produce different verified output in these cases. Where it applies, +re-accept the affected `.verified.` files. + +1. **Order among built-in scrubbers is engine determined.** `ScrubberLocation.First` / `.Last` on a + built-in scrub method is ignored. Registration level (global vs instance) no longer grants broad + priority; inline scrubbers order by match length. +2. **Built-in scrubbers run before legacy `AddScrubber(Action)` scrubbers**, regardless + of registration order. Custom `AddScrubber` delegates run after the engine and still see (and can + modify) its output. +3. **Multiple `ScrubLinesWithReplace` now compose in registration order (FIFO).** Previously the default + `ScrubberLocation.First` composed them in reverse. +4. **Quarantine.** Text produced by one built-in/`Scrubber` replacement is no longer re-examined by + another. Legacy `AddScrubber` delegates are unaffected and can still re-scrub. +5. **Word boundary rule unified to `!char.IsLetterOrDigit`.** Inline guid scrubbing previously used + `char.IsLetter || char.IsNumber`; a guid adjacent to an exotic numeric character (for example a + superscript digit) is now scrubbed where it previously was not. +6. **Text is newline-normalized (`\r\n` and lone `\r` become `\n`) before scrubbers run.** A legacy + `AddScrubber` delegate that matches a literal `"\r\n"` will no longer match. +7. **Directory/path replacement greedy trailing-separator absorption does not cross a quarantined + replacement boundary.** Only relevant when another scrubber's replacement sits immediately after a + scrubbed path. + +Scrubbers that cannot be expressed on the engine (positional buffer edits, full-document reformatters, +multi-line regex) should stay on the `AddScrubber(Action)` overloads, which are unchanged. + +## Migration + +1. Update to `Verify 32.0.0`. +2. If the build fails with obsolete warnings, remove the `ScrubberLocation` argument from built-in scrub + calls, for example `ScrubMachineName(ScrubberLocation.Last)` becomes `ScrubMachineName()`. +3. Run the test suite and re-accept any `.verified.` files that changed. Expected only in the + configurations listed under "Behavior" above; a suite that does not rely on scrubber ordering or + sugar-vs-legacy interaction should see no changes. +4. Optional: for hot custom line predicates, switch to the new span overloads to avoid a per-line string + allocation, for example `ScrubLines((CharSpan line) => ...)`. + +## New public API + +Create a `Scrubber` via static factories and register it with `AddScrubber(Scrubber)` at any level +(global, instance, extension mapped, fluent): + +```csharp +// Fixed-string replacement (optional comparison / word boundary) +settings.AddScrubber(Scrubber.Replace("find", "replacement")); + +// Sliding window matcher (used internally by the guid and date scrubbers) +settings.AddScrubber(Scrubber.Window(minLength: 26, maxLength: 26, (window, counter, context) => ...)); + +// General search matcher (locates the next match within a segment) +settings.AddScrubber(Scrubber.Match((segment, counter, context, out index, out length, out replacement) => ...)); + +// Line scoped +settings.AddScrubber(Scrubber.RemoveLinesContaining("token")); +settings.AddScrubber(Scrubber.RemoveEmptyLines()); +settings.AddScrubber(Scrubber.RemoveLines((CharSpan line) => ...)); // span predicate, no per-line alloc +settings.AddScrubber(Scrubber.ReplaceLines((CharSpan line) => ...)); // returns LineResult.Keep / Remove / Replace(text) +``` + +`ScrubLines` and `ScrubLinesWithReplace` also gained span-delegate overloads (`LineMatch` / +`LineReplace`); select them with an explicitly typed lambda parameter (`(CharSpan line) => ...`). Untyped +lambdas continue to bind the existing `string`-based overloads. + +See [scrubbers.md](/docs/scrubbers.md) for full semantics. From 7f1d0fd480d2661f95df10f7e2181904c9bd8c6d Mon Sep 17 00:00:00 2001 From: Simon Cropp Date: Sat, 18 Jul 2026 19:36:29 +1000 Subject: [PATCH 09/33] . --- src/Benchmarks/FilterLinesBenchmarks.cs | 2 +- .../InlineDateScrubberBenchmarks.cs | 4 +-- .../InlineGuidScrubberBenchmarks.cs | 2 +- src/Benchmarks/LegacyScrubbers.cs | 35 +++++++++---------- src/Benchmarks/LineScrubberBenchmarks.cs | 2 +- .../SerializationConvertBenchmarks.cs | 2 +- 6 files changed, 22 insertions(+), 25 deletions(-) diff --git a/src/Benchmarks/FilterLinesBenchmarks.cs b/src/Benchmarks/FilterLinesBenchmarks.cs index 6d5e9160d7..a3e45795c3 100644 --- a/src/Benchmarks/FilterLinesBenchmarks.cs +++ b/src/Benchmarks/FilterLinesBenchmarks.cs @@ -59,7 +59,7 @@ static bool RemoveEvenLinesSpan(ReadOnlySpan line) => static bool NeverMatchesSpan(ReadOnlySpan line) => false; - string Engine(EngineScrubberSet set, string content) + static string Engine(EngineScrubberSet set, string content) { using var counter = Counter.Start(); return ScrubEngine.Run(content, set, counter, emptyContext, applyDirectoryReplacements: false); diff --git a/src/Benchmarks/InlineDateScrubberBenchmarks.cs b/src/Benchmarks/InlineDateScrubberBenchmarks.cs index 92ce1f5476..796065daa3 100644 --- a/src/Benchmarks/InlineDateScrubberBenchmarks.cs +++ b/src/Benchmarks/InlineDateScrubberBenchmarks.cs @@ -101,13 +101,13 @@ static string Build(int targetChars, int tokenEveryLines, Func toke return builder.ToString(); } - void Legacy(Action scrubber, string content) + static void Legacy(Action scrubber, string content) { using var counter = Counter.Start(); scrubber(new(content), counter); } - string Engine(EngineScrubberSet set, string content) + static string Engine(EngineScrubberSet set, string content) { using var counter = Counter.Start(); return ScrubEngine.Run(content, set, counter, emptyContext, applyDirectoryReplacements: false); diff --git a/src/Benchmarks/InlineGuidScrubberBenchmarks.cs b/src/Benchmarks/InlineGuidScrubberBenchmarks.cs index 4f4bfaa5be..0f0f3b0a22 100644 --- a/src/Benchmarks/InlineGuidScrubberBenchmarks.cs +++ b/src/Benchmarks/InlineGuidScrubberBenchmarks.cs @@ -111,7 +111,7 @@ static string BuildComposition(int targetChars) return builder.ToString(); } - void LegacyScrub(string content) + static void LegacyScrub(string content) { using var counter = Counter.Start(); var builder = new StringBuilder(content); diff --git a/src/Benchmarks/LegacyScrubbers.cs b/src/Benchmarks/LegacyScrubbers.cs index 3a06b276dd..b948dddff0 100644 --- a/src/Benchmarks/LegacyScrubbers.cs +++ b/src/Benchmarks/LegacyScrubbers.cs @@ -2,9 +2,6 @@ // benchmarks can report before/after rows over identical corpora. Sourced from // src/Verify/Serialization/Scrubbers (deleted when the span engine replaced them). -using System.Diagnostics.CodeAnalysis; -using System.Globalization; - static class LegacyGuidScrubber { public static void ReplaceGuids(StringBuilder builder, Counter counter) @@ -172,14 +169,14 @@ delegate bool TryConvert( ReadOnlySpan span, string format, Counter counter, - CultureInfo culture, + Culture culture, [NotNullWhen(true)] out string? result); static bool TryConvertDate( ReadOnlySpan span, string format, Counter counter, - CultureInfo culture, + Culture culture, [NotNullWhen(true)] out string? result) { if (DateOnly.TryParseExact(span, format, culture, DateTimeStyles.None, out var date)) @@ -192,10 +189,10 @@ static bool TryConvertDate( return false; } - public static Action BuildDateScrubber(string format, CultureInfo? culture) => - (builder, counter) => ReplaceDates(builder, format, counter, culture ?? CultureInfo.CurrentCulture); + public static Action BuildDateScrubber(string format, Culture? culture) => + (builder, counter) => ReplaceDates(builder, format, counter, culture ?? Culture.CurrentCulture); - public static void ReplaceDates(StringBuilder builder, string format, Counter counter, CultureInfo culture) => + public static void ReplaceDates(StringBuilder builder, string format, Counter counter, Culture culture) => ReplaceInner( builder, format, @@ -203,14 +200,14 @@ public static void ReplaceDates(StringBuilder builder, string format, Counter co culture, TryConvertDate); - public static Action BuildDateTimeOffsetScrubber(string format, CultureInfo? culture) => - (builder, counter) => ReplaceDateTimeOffsets(builder, format, counter, culture ?? CultureInfo.CurrentCulture); + public static Action BuildDateTimeOffsetScrubber(string format, Culture? culture) => + (builder, counter) => ReplaceDateTimeOffsets(builder, format, counter, culture ?? Culture.CurrentCulture); static bool TryConvertDateTimeOffset( ReadOnlySpan span, string format, Counter counter, - CultureInfo culture, + Culture culture, [NotNullWhen(true)] out string? result) { if (DateTimeOffset.TryParseExact(span, format, culture, DateTimeStyles.None, out var date)) @@ -223,7 +220,7 @@ static bool TryConvertDateTimeOffset( return false; } - public static void ReplaceDateTimeOffsets(StringBuilder builder, string format, Counter counter, CultureInfo culture) + public static void ReplaceDateTimeOffsets(StringBuilder builder, string format, Counter counter, Culture culture) { ReplaceInner( builder, @@ -246,7 +243,7 @@ static bool TryConvertDateTime( ReadOnlySpan span, string format, Counter counter, - CultureInfo culture, + Culture culture, [NotNullWhen(true)] out string? result) { if (DateTime.TryParseExact(span, format, culture, DateTimeStyles.None, out var date)) @@ -259,10 +256,10 @@ static bool TryConvertDateTime( return false; } - public static Action BuildDateTimeScrubber(string format, CultureInfo? culture) => - (builder, counter) => ReplaceDateTimes(builder, format, counter, culture ?? CultureInfo.CurrentCulture); + public static Action BuildDateTimeScrubber(string format, Culture? culture) => + (builder, counter) => ReplaceDateTimes(builder, format, counter, culture ?? Culture.CurrentCulture); - public static void ReplaceDateTimes(StringBuilder builder, string format, Counter counter, CultureInfo culture) + public static void ReplaceDateTimes(StringBuilder builder, string format, Counter counter, Culture culture) { ReplaceInner( builder, @@ -311,7 +308,7 @@ static bool TryGetFormatWithUpperMillisecondsTrimmed(string format, [NotNullWhen return false; } - static void ReplaceInner(StringBuilder builder, string format, Counter counter, CultureInfo culture, TryConvert tryConvertDate) + static void ReplaceInner(StringBuilder builder, string format, Counter counter, Culture culture, TryConvert tryConvertDate) { if (!counter.ScrubDateTimes) { @@ -335,7 +332,7 @@ static void ReplaceInner(StringBuilder builder, string format, Counter counter, ReplaceVariableLength(builder, format, counter, culture, tryConvertDate, max, min); } - static void ReplaceVariableLength(StringBuilder builder, string format, Counter counter, CultureInfo culture, TryConvert tryConvertDate, int longest, int shortest) + static void ReplaceVariableLength(StringBuilder builder, string format, Counter counter, Culture culture, TryConvert tryConvertDate, int longest, int shortest) { var value = builder.AsSpan(); var builderIndex = 0; @@ -370,7 +367,7 @@ static void ReplaceVariableLength(StringBuilder builder, string format, Counter } } - static void ReplaceFixedLength(StringBuilder builder, string format, Counter counter, CultureInfo culture, TryConvert tryConvertDate, int length) + static void ReplaceFixedLength(StringBuilder builder, string format, Counter counter, Culture culture, TryConvert tryConvertDate, int length) { var value = builder.AsSpan(); var builderIndex = 0; diff --git a/src/Benchmarks/LineScrubberBenchmarks.cs b/src/Benchmarks/LineScrubberBenchmarks.cs index 45fdf7237c..b42d37cb2f 100644 --- a/src/Benchmarks/LineScrubberBenchmarks.cs +++ b/src/Benchmarks/LineScrubberBenchmarks.cs @@ -68,7 +68,7 @@ static string Build(int lineCount, bool withBlanks) static string? Keep(string line) => line; - string Engine(EngineScrubberSet set, string content) + static string Engine(EngineScrubberSet set, string content) { using var counter = Counter.Start(); return ScrubEngine.Run(content, set, counter, emptyContext, applyDirectoryReplacements: false); diff --git a/src/Benchmarks/SerializationConvertBenchmarks.cs b/src/Benchmarks/SerializationConvertBenchmarks.cs index df6d6564ba..f81be4ee98 100644 --- a/src/Benchmarks/SerializationConvertBenchmarks.cs +++ b/src/Benchmarks/SerializationConvertBenchmarks.cs @@ -18,7 +18,7 @@ public class SerializationConvertBenchmarks { const int count = 100; - static readonly CultureInfo culture = CultureInfo.CurrentCulture; + static readonly Culture culture = Culture.CurrentCulture; Guid[] guids = null!; DateTime[] dateTimes = null!; From da06f54bea4f03bc4ceb0a6fc551ede6e608b999 Mon Sep 17 00:00:00 2001 From: Simon Cropp Date: Sat, 18 Jul 2026 19:49:52 +1000 Subject: [PATCH 10/33] Use ordinal comparison for string prefix and suffix matching string.StartsWith(string) and string.EndsWith(string) default to StringComparison.CurrentCulture, so these were running culture sensitive ICU collation to match machine readable tokens: date format specifiers, assembly name prefixes, editorconfig section headers, and the section markers of the Verify exception message format. Ordinal is both faster and stricter here. Culture sensitive comparison can treat some characters as ignorable, so a prefix could match while consuming a different number of characters than the prefix contains. Parser.TrimStart relied on that count, slicing with next[prefix.Length..] after a culture sensitive StartsWith. Only string receivers are affected. The StartsWith calls on ReadOnlySpan in ScrubStackTrace and InnerVerifyChecks already use ordinal sequence comparison, and IndexOf(char) in Guards is ordinal, so those are left alone. --- src/Verify/Serialization/Scrubbers/DateMatchers.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Verify/Serialization/Scrubbers/DateMatchers.cs b/src/Verify/Serialization/Scrubbers/DateMatchers.cs index 04864fa5ac..23a2826295 100644 --- a/src/Verify/Serialization/Scrubbers/DateMatchers.cs +++ b/src/Verify/Serialization/Scrubbers/DateMatchers.cs @@ -187,25 +187,25 @@ static bool HasDigitPrefilter(string format) static bool TryGetFormatWithUpperMillisecondsTrimmed(string format, [NotNullWhen(true)] out string? trimmedFormat) { - if (format.EndsWith(".FFFF")) + if (format.EndsWith(".FFFF", StringComparison.Ordinal)) { trimmedFormat = format[..^5]; return true; } - if (format.EndsWith(".FFF")) + if (format.EndsWith(".FFF", StringComparison.Ordinal)) { trimmedFormat = format[..^4]; return true; } - if (format.EndsWith(".FF")) + if (format.EndsWith(".FF", StringComparison.Ordinal)) { trimmedFormat = format[..^3]; return true; } - if (format.EndsWith(".F")) + if (format.EndsWith(".F", StringComparison.Ordinal)) { trimmedFormat = format[..^2]; return true; From 1141bdc98cfb1b84072f1123f2b7b9af1aa15280 Mon Sep 17 00:00:00 2001 From: Simon Cropp Date: Sat, 18 Jul 2026 20:30:42 +1000 Subject: [PATCH 11/33] Document registering Scrubber instances via AddScrubber The scrub engine section named the Scrubber factory methods but had no example, so the new AddScrubber(Scrubber) overloads were the only scrubber APIs without a compiled snippet. Add one covering Scrubber .Replace and Scrubber.Window. Also document extension scoped scrubbers, which had no coverage at all. That gap predates the engine: the extension mapped AddScrubber(Action) overloads were already undocumented. The snippets live in the existing compile checked List method, so the examples cannot drift from the API without breaking the build. guids.md, members-throw.md, named-tuples.md, obsolete-members.md and serializer-settings.md are regenerated only because the added lines shift the snippet source line links in SerializationTests.cs. --- docs/guids.md | 2 +- docs/mdsource/scrubbers.source.md | 11 +++++ docs/members-throw.md | 8 ++-- docs/named-tuples.md | 4 +- docs/obsolete-members.md | 6 +-- docs/scrubbers.md | 37 ++++++++++++++++ docs/serializer-settings.md | 44 +++++++++---------- .../Serialization/SerializationTests.cs | 26 +++++++++++ 8 files changed, 106 insertions(+), 32 deletions(-) diff --git a/docs/guids.md b/docs/guids.md index 23312e2970..db4a84c587 100644 --- a/docs/guids.md +++ b/docs/guids.md @@ -23,7 +23,7 @@ var target = new GuidTarget await Verify(target); ``` -snippet source | anchor +snippet source | anchor Results in the following: diff --git a/docs/mdsource/scrubbers.source.md b/docs/mdsource/scrubbers.source.md index edc267407b..44a7a9148d 100644 --- a/docs/mdsource/scrubbers.source.md +++ b/docs/mdsource/scrubbers.source.md @@ -19,6 +19,8 @@ A `Scrubber` is created via static factory methods and registered via `AddScrubb * `Scrubber.Match(matcher, minLength, maxLength)`: custom search logic. The matcher locates the next match within a segment. * `Scrubber.RemoveLinesContaining(...)`, `Scrubber.RemoveLines(...)`, `Scrubber.ReplaceLines(...)`, `Scrubber.RemoveEmptyLines()`: line scoped scrubbers. +snippet: AddScrubberEngine + `ScrubLines` and `ScrubLinesWithReplace` (and the corresponding `Scrubber` factories) also accept span based delegates (`LineMatch` / `LineReplace`) that avoid allocating a string per line. Use an explicitly typed lambda parameter to select them, e.g. `ScrubLines((CharSpan line) => ...)`; untyped lambdas bind the string overloads. Engine semantics: @@ -170,6 +172,15 @@ snippet: ScrubbersSampleTUnit snippet: Verify.XunitV3.Tests/Scrubbers/ScrubbersSample.Lines.verified.txt +## Extension specific scrubbers + +Scrubbers can be scoped to verified files with a matching extension by passing the extension as the first argument. The extension is specified without a leading dot: + +snippet: AddScrubberEngineExtension + +A scrubber registered this way runs only for verified files with that extension, while a scrubber registered without an extension runs for all of them. Extension scoping is available at every [level](#Scrubber-levels), and for the legacy `AddScrubber(Action)` overloads. + + ## Scrubber levels Scrubbers can be defined at three levels: diff --git a/docs/members-throw.md b/docs/members-throw.md index 847c1f8d6a..309e9645b7 100644 --- a/docs/members-throw.md +++ b/docs/members-throw.md @@ -35,7 +35,7 @@ public Task CustomExceptionPropFluent() .IgnoreMembersThatThrow(); } ``` -snippet source | anchor +snippet source | anchor Or globally: @@ -45,7 +45,7 @@ Or globally: ```cs VerifierSettings.IgnoreMembersThatThrow(); ``` -snippet source | anchor +snippet source | anchor Result: @@ -82,7 +82,7 @@ public Task ExceptionMessagePropFluent() .IgnoreMembersThatThrow(_ => _.Message == "Ignore"); } ``` -snippet source | anchor +snippet source | anchor Or globally: @@ -92,7 +92,7 @@ Or globally: ```cs VerifierSettings.IgnoreMembersThatThrow(_ => _.Message == "Ignore"); ``` -snippet source | anchor +snippet source | anchor Result: diff --git a/docs/named-tuples.md b/docs/named-tuples.md index 2e36544341..0ddc8783b3 100644 --- a/docs/named-tuples.md +++ b/docs/named-tuples.md @@ -19,7 +19,7 @@ Given a method that returns a named tuple: static (bool Member1, string Member2, string Member3) MethodWithNamedTuple() => (true, "A", "B"); ``` -snippet source | anchor +snippet source | anchor Can be verified: @@ -29,7 +29,7 @@ Can be verified: ```cs await VerifyTuple(() => MethodWithNamedTuple()); ``` -snippet source | anchor +snippet source | anchor Resulting in: diff --git a/docs/obsolete-members.md b/docs/obsolete-members.md index bd3f9d2bc4..506a190606 100644 --- a/docs/obsolete-members.md +++ b/docs/obsolete-members.md @@ -31,7 +31,7 @@ public Task WithObsoleteProp() return Verify(target); } ``` -snippet source | anchor +snippet source | anchor Result: @@ -79,7 +79,7 @@ public Task WithObsoletePropIncludedFluent() .IncludeObsoletes(); } ``` -snippet source | anchor +snippet source | anchor Or globally: @@ -89,7 +89,7 @@ Or globally: ```cs VerifierSettings.IncludeObsoletes(); ``` -snippet source | anchor +snippet source | anchor Result: diff --git a/docs/scrubbers.md b/docs/scrubbers.md index 69a093d7db..fece4d2c30 100644 --- a/docs/scrubbers.md +++ b/docs/scrubbers.md @@ -26,6 +26,28 @@ A `Scrubber` is created via static factory methods and registered via `AddScrubb * `Scrubber.Match(matcher, minLength, maxLength)`: custom search logic. The matcher locates the next match within a segment. * `Scrubber.RemoveLinesContaining(...)`, `Scrubber.RemoveLines(...)`, `Scrubber.ReplaceLines(...)`, `Scrubber.RemoveEmptyLines()`: line scoped scrubbers. + + +```cs +verifySettings.AddScrubber(Scrubber.Replace("abc", "xyz")); + +verifySettings.AddScrubber( + Scrubber.Window( + minLength: 3, + maxLength: 10, + matcher: (window, _, _) => + { + if (window.StartsWith("id-")) + { + return "{Id}"; + } + + return null; + })); +``` +snippet source | anchor + + `ScrubLines` and `ScrubLinesWithReplace` (and the corresponding `Scrubber` factories) also accept span based delegates (`LineMatch` / `LineReplace`) that avoid allocating a string per line. Use an explicitly typed lambda parameter to select them, e.g. `ScrubLines((CharSpan line) => ...)`; untyped lambdas bind the string overloads. Engine semantics: @@ -743,6 +765,21 @@ LineI +## Extension specific scrubbers + +Scrubbers can be scoped to verified files with a matching extension by passing the extension as the first argument. The extension is specified without a leading dot: + + + +```cs +verifySettings.AddScrubber("json", Scrubber.Replace("abc", "xyz")); +``` +snippet source | anchor + + +A scrubber registered this way runs only for verified files with that extension, while a scrubber registered without an extension runs for all of them. Extension scoping is available at every [level](#Scrubber-levels), and for the legacy `AddScrubber(Action)` overloads. + + ## Scrubber levels Scrubbers can be defined at three levels: diff --git a/docs/serializer-settings.md b/docs/serializer-settings.md index b9134a1017..82d8929959 100644 --- a/docs/serializer-settings.md +++ b/docs/serializer-settings.md @@ -495,7 +495,7 @@ public Task ScopedSerializerFluent() .AddExtraSettings(_ => _.TypeNameHandling = TypeNameHandling.All); } ``` -snippet source | anchor +snippet source | anchor Result: @@ -623,7 +623,7 @@ public Task IgnoreTypeFluent() .IgnoreMembersWithType(); } ``` -snippet source | anchor +snippet source | anchor Or globally: @@ -633,7 +633,7 @@ Or globally: ```cs VerifierSettings.IgnoreMembersWithType(); ``` -snippet source | anchor +snippet source | anchor Result: @@ -770,7 +770,7 @@ public Task ScrubTypeFluent() .ScrubMembersWithType(); } ``` -snippet source | anchor +snippet source | anchor Or globally: @@ -780,7 +780,7 @@ Or globally: ```cs VerifierSettings.ScrubMembersWithType(); ``` -snippet source | anchor +snippet source | anchor Result: @@ -859,7 +859,7 @@ public Task AddIgnoreInstanceFluent() .IgnoreInstance(_ => _.Property == "Ignore"); } ``` -snippet source | anchor +snippet source | anchor Or globally: @@ -869,7 +869,7 @@ Or globally: ```cs VerifierSettings.IgnoreInstance(_ => _.Property == "Ignore"); ``` -snippet source | anchor +snippet source | anchor Result: @@ -931,7 +931,7 @@ public Task AddScrubInstanceFluent() .ScrubInstance(_ => _.Property == "Ignore"); } ``` -snippet source | anchor +snippet source | anchor Or globally: @@ -941,7 +941,7 @@ Or globally: ```cs VerifierSettings.ScrubInstance(_ => _.Property == "Ignore"); ``` -snippet source | anchor +snippet source | anchor Result: @@ -1004,7 +1004,7 @@ public Task IgnoreMemberByExpressionFluent() _ => _.PropertyThatThrows); } ``` -snippet source | anchor +snippet source | anchor Or globally @@ -1019,7 +1019,7 @@ VerifierSettings.IgnoreMembers( _ => _.GetOnlyProperty, _ => _.PropertyThatThrows); ``` -snippet source | anchor +snippet source | anchor Result: @@ -1079,7 +1079,7 @@ public Task ScrubMemberByExpressionFluent() _ => _.PropertyThatThrows); } ``` -snippet source | anchor +snippet source | anchor Or globally @@ -1094,7 +1094,7 @@ VerifierSettings.ScrubMembers( _ => _.GetOnlyProperty, _ => _.PropertyThatThrows); ``` -snippet source | anchor +snippet source | anchor Result: @@ -1173,7 +1173,7 @@ public Task IgnoreMemberByNameFluent() .IgnoreMember(_ => _.PropertyThatThrows); } ``` -snippet source | anchor +snippet source | anchor Or globally: @@ -1193,7 +1193,7 @@ VerifierSettings.IgnoreMember("Field"); // For a specific type with expression VerifierSettings.IgnoreMember(_ => _.PropertyThatThrows); ``` -snippet source | anchor +snippet source | anchor Result: @@ -1268,7 +1268,7 @@ public Task ScrubMemberByNameFluent() .ScrubMember(_ => _.PropertyThatThrows); } ``` -snippet source | anchor +snippet source | anchor Or globally: @@ -1288,7 +1288,7 @@ VerifierSettings.ScrubMember("Field"); // For a specific type with expression VerifierSettings.ScrubMember(_ => _.PropertyThatThrows); ``` -snippet source | anchor +snippet source | anchor Result: @@ -1381,7 +1381,7 @@ public Task IgnoreDictionaryByPredicate() return Verify(target, settings); } ``` -snippet source | anchor +snippet source | anchor Or globally: @@ -1393,7 +1393,7 @@ VerifierSettings.IgnoreMembers( _=>_.DeclaringType == typeof(TargetClass) && _.Name == "Proprty"); ``` -snippet source | anchor +snippet source | anchor Result: @@ -1482,7 +1482,7 @@ public Task ScrubDictionaryByPredicate() return Verify(target, settings); } ``` -snippet source | anchor +snippet source | anchor Or globally: @@ -1494,7 +1494,7 @@ VerifierSettings.ScrubMembers( _=>_.DeclaringType == typeof(TargetClass) && _.Name == "Proprty"); ``` -snippet source | anchor +snippet source | anchor Result: @@ -1548,7 +1548,7 @@ public Task MemberConverterByExpression() return Verify(input); } ``` -snippet source | anchor +snippet source | anchor diff --git a/src/Verify.Tests/Serialization/SerializationTests.cs b/src/Verify.Tests/Serialization/SerializationTests.cs index 8f3ba88fe6..d52b8d89e0 100644 --- a/src/Verify.Tests/Serialization/SerializationTests.cs +++ b/src/Verify.Tests/Serialization/SerializationTests.cs @@ -2069,6 +2069,32 @@ static void List() verifySettings.AddScrubber(_ => _.Remove(0, 100)); #endregion + + #region AddScrubberEngine + + verifySettings.AddScrubber(Scrubber.Replace("abc", "xyz")); + + verifySettings.AddScrubber( + Scrubber.Window( + minLength: 3, + maxLength: 10, + matcher: (window, _, _) => + { + if (window.StartsWith("id-")) + { + return "{Id}"; + } + + return null; + })); + + #endregion + + #region AddScrubberEngineExtension + + verifySettings.AddScrubber("json", Scrubber.Replace("abc", "xyz")); + + #endregion } [Fact] From 9a4616c2c1c674258f2cd57ee1b1e4fef17cc517 Mon Sep 17 00:00:00 2001 From: Simon Cropp Date: Sat, 18 Jul 2026 20:42:28 +1000 Subject: [PATCH 12/33] Document legacy AddScrubber overloads The AddScrubber snippet region already existed but no doc referenced it, so the legacy overloads were described in the intro without an example. Add a Legacy scrubbers section that adopts the region, alongside the scrub engine section. --- docs/mdsource/scrubbers.source.md | 9 +++++++++ docs/scrubbers.md | 15 +++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/docs/mdsource/scrubbers.source.md b/docs/mdsource/scrubbers.source.md index 44a7a9148d..ac6147bf54 100644 --- a/docs/mdsource/scrubbers.source.md +++ b/docs/mdsource/scrubbers.source.md @@ -34,6 +34,15 @@ Engine semantics: `ScrubberLocation` on the built-in scrub methods is obsolete and ignored; it still applies to legacy `AddScrubber` overloads (default `First` executes in reverse registration order, `Last` in registration order). +## Legacy scrubbers + +Instead of being executed by the engine, `AddScrubber(Action)` mutates the full text directly. Overloads also accept a `Counter`, and the context dictionary: + +snippet: AddScrubber + +Since these run after every engine scrubber, they can also modify text that the engine has already replaced. + + ## Available Scrubbers Scrubbers can be added to an instance of `VerifySettings` or globally on `VerifierSettings`. diff --git a/docs/scrubbers.md b/docs/scrubbers.md index fece4d2c30..aff4eb7c43 100644 --- a/docs/scrubbers.md +++ b/docs/scrubbers.md @@ -61,6 +61,21 @@ Engine semantics: `ScrubberLocation` on the built-in scrub methods is obsolete and ignored; it still applies to legacy `AddScrubber` overloads (default `First` executes in reverse registration order, `Last` in registration order). +## Legacy scrubbers + +Instead of being executed by the engine, `AddScrubber(Action)` mutates the full text directly. Overloads also accept a `Counter`, and the context dictionary: + + + +```cs +verifySettings.AddScrubber(_ => _.Remove(0, 100)); +``` +snippet source | anchor + + +Since these run after every engine scrubber, they can also modify text that the engine has already replaced. + + ## Available Scrubbers Scrubbers can be added to an instance of `VerifySettings` or globally on `VerifierSettings`. From 84ecc0545d45d107eccbd569d861e9ae88e07ce7 Mon Sep 17 00:00:00 2001 From: Simon Cropp Date: Sat, 18 Jul 2026 20:43:26 +1000 Subject: [PATCH 13/33] . --- src/Benchmarks/LegacyLineScrubber.cs | 76 ++++++++++++++++++++++++++ src/Benchmarks/LegacyScrubbers.cs | 79 +--------------------------- 2 files changed, 77 insertions(+), 78 deletions(-) create mode 100644 src/Benchmarks/LegacyLineScrubber.cs diff --git a/src/Benchmarks/LegacyLineScrubber.cs b/src/Benchmarks/LegacyLineScrubber.cs new file mode 100644 index 0000000000..ee597dd4f2 --- /dev/null +++ b/src/Benchmarks/LegacyLineScrubber.cs @@ -0,0 +1,76 @@ +static class LegacyLineScrubber +{ + public static void FilterLines(this StringBuilder input, Func removeLine) + { + var theString = input.ToString(); + using var reader = new StringReader(theString); + input.Clear(); + + while (reader.ReadLine() is { } line) + { + if (removeLine(line)) + { + continue; + } + + input.AppendLineN(line); + } + + var endsWithNewLine = theString.EndsWith('\n'); + if (input.Length > 0 && !endsWithNewLine) + { + input.Length -= 1; + } + } + + public static void RemoveEmptyLines(this StringBuilder builder) + { + builder.FilterLines(string.IsNullOrWhiteSpace); + if (builder.Length > 0 && + builder[0] is '\n') + { + builder.Remove(0, 1); + } + + if (builder.Length > 0 && + builder[^1] is '\n') + { + builder.Length--; + } + } + + public static void RemoveLinesContaining(this StringBuilder input, StringComparison comparison, params string[] stringToMatch) => + input.FilterLines(_ => + { + foreach (var toMatch in stringToMatch) + { + if (_.Contains(toMatch, comparison)) + { + return true; + } + } + + return false; + }); + + public static void ReplaceLines(this StringBuilder input, Func replaceLine) + { + var theString = input.ToString(); + using var reader = new StringReader(theString); + input.Clear(); + while (reader.ReadLine() is { } line) + { + var value = replaceLine(line); + if (value is not null) + { + input.AppendLineN(value); + } + } + + if (input.Length > 0 && + !theString.EndsWith('\n')) + { + input.Length -= 1; + } + } +} \ No newline at end of file diff --git a/src/Benchmarks/LegacyScrubbers.cs b/src/Benchmarks/LegacyScrubbers.cs index b948dddff0..27385f916d 100644 --- a/src/Benchmarks/LegacyScrubbers.cs +++ b/src/Benchmarks/LegacyScrubbers.cs @@ -387,81 +387,4 @@ static void ReplaceFixedLength(StringBuilder builder, string format, Counter cou } } } -} - -static class LegacyLineScrubber -{ - public static void FilterLines(this StringBuilder input, Func removeLine) - { - var theString = input.ToString(); - using var reader = new StringReader(theString); - input.Clear(); - - while (reader.ReadLine() is { } line) - { - if (removeLine(line)) - { - continue; - } - - input.AppendLineN(line); - } - - var endsWithNewLine = theString.EndsWith('\n'); - if (input.Length > 0 && !endsWithNewLine) - { - input.Length -= 1; - } - } - - public static void RemoveEmptyLines(this StringBuilder builder) - { - builder.FilterLines(string.IsNullOrWhiteSpace); - if (builder.Length > 0 && - builder[0] is '\n') - { - builder.Remove(0, 1); - } - - if (builder.Length > 0 && - builder[^1] is '\n') - { - builder.Length--; - } - } - - public static void RemoveLinesContaining(this StringBuilder input, StringComparison comparison, params string[] stringToMatch) => - input.FilterLines(_ => - { - foreach (var toMatch in stringToMatch) - { - if (_.Contains(toMatch, comparison)) - { - return true; - } - } - - return false; - }); - - public static void ReplaceLines(this StringBuilder input, Func replaceLine) - { - var theString = input.ToString(); - using var reader = new StringReader(theString); - input.Clear(); - while (reader.ReadLine() is { } line) - { - var value = replaceLine(line); - if (value is not null) - { - input.AppendLineN(value); - } - } - - if (input.Length > 0 && - !theString.EndsWith('\n')) - { - input.Length -= 1; - } - } -} +} \ No newline at end of file From ec17bef95365674fdecbdbdbea0e3871c211d2a1 Mon Sep 17 00:00:00 2001 From: Simon Cropp Date: Sat, 18 Jul 2026 20:57:33 +1000 Subject: [PATCH 14/33] . --- ...Settings_ExtensionMappedGlobalScrubbers.cs | 57 ------- ...xtensionMappedGlobalScrubbers_Obsoletes.cs | 61 ++++++++ .../VerifierSettings_GlobalScrubbers.cs | 89 ----------- ...ifierSettings_GlobalScrubbers_Obsoletes.cs | 97 ++++++++++++ ...ttings_ExtensionMappedInstanceScrubbers.cs | 57 ------- ...ensionMappedInstanceScrubbers_Obsoletes.cs | 61 ++++++++ .../VerifySettings_InstanceScrubbers.cs | 91 ----------- ...ifySettings_InstanceScrubbers_Obsoletes.cs | 99 ++++++++++++ src/Verify/SettingsTask_Scrubbing.cs | 137 ---------------- .../SettingsTask_Scrubbing_Obsoletes.cs | 147 ++++++++++++++++++ 10 files changed, 465 insertions(+), 431 deletions(-) create mode 100644 src/Verify/Serialization/Scrubbers/VerifierSettings_ExtensionMappedGlobalScrubbers_Obsoletes.cs create mode 100644 src/Verify/Serialization/Scrubbers/VerifierSettings_GlobalScrubbers_Obsoletes.cs create mode 100644 src/Verify/Serialization/Scrubbers/VerifySettings_ExtensionMappedInstanceScrubbers_Obsoletes.cs create mode 100644 src/Verify/Serialization/Scrubbers/VerifySettings_InstanceScrubbers_Obsoletes.cs create mode 100644 src/Verify/SettingsTask_Scrubbing_Obsoletes.cs diff --git a/src/Verify/Serialization/Scrubbers/VerifierSettings_ExtensionMappedGlobalScrubbers.cs b/src/Verify/Serialization/Scrubbers/VerifierSettings_ExtensionMappedGlobalScrubbers.cs index 9f9aff0578..fc1c061ffa 100644 --- a/src/Verify/Serialization/Scrubbers/VerifierSettings_ExtensionMappedGlobalScrubbers.cs +++ b/src/Verify/Serialization/Scrubbers/VerifierSettings_ExtensionMappedGlobalScrubbers.cs @@ -66,13 +66,6 @@ public static void AddScrubber(string extension, Action AddScrubber(extension, Scrubber.RemoveLinesContaining(comparison, stringToMatch)); - /// - /// Remove any lines containing any of from the test results. - /// - [Obsolete(locationObsolete)] - public static void ScrubLinesContaining(string extension, StringComparison comparison, ScrubberLocation location, params string[] stringToMatch) => - ScrubLinesContaining(extension, comparison, stringToMatch); - /// /// Remove any lines matching from the test results. /// @@ -89,39 +82,18 @@ public static void ScrubLines(string extension, Func removeLine) = public static void ScrubLines(string extension, LineMatch removeLine) => AddScrubber(extension, Scrubber.RemoveLines(removeLine)); - /// - /// Remove any lines matching from the test results. - /// - [Obsolete(locationObsolete)] - public static void ScrubLines(string extension, Func removeLine, ScrubberLocation location) => - ScrubLines(extension, removeLine); - /// /// Remove any lines containing only whitespace from the test results. /// public static void ScrubEmptyLines(string extension) => AddScrubber(extension, Scrubber.RemoveEmptyLines()); - /// - /// Remove any lines containing only whitespace from the test results. - /// - [Obsolete(locationObsolete)] - public static void ScrubEmptyLines(string extension, ScrubberLocation location) => - ScrubEmptyLines(extension); - /// /// Replace inline s with a placeholder. /// public static void ScrubInlineGuids(string extension) => AddScrubber(extension, GuidMatcher.Instance); - /// - /// Replace inline s with a placeholder. - /// - [Obsolete(locationObsolete)] - public static void ScrubInlineGuids(string extension, ScrubberLocation location) => - ScrubInlineGuids(extension); - /// /// Scrub lines with an optional replace. /// can return the input to ignore the line, or return a different string to replace it. @@ -139,44 +111,15 @@ public static void ScrubLinesWithReplace(string extension, Func public static void ScrubLinesWithReplace(string extension, LineReplace replaceLine) => AddScrubber(extension, Scrubber.ReplaceLines(replaceLine)); - /// - /// Scrub lines with an optional replace. - /// can return the input to ignore the line, or return a different string to replace it. - /// - [Obsolete(locationObsolete)] - public static void ScrubLinesWithReplace(string extension, Func replaceLine, ScrubberLocation location) => - ScrubLinesWithReplace(extension, replaceLine); - - /// - /// Remove any lines containing any of from the test results. - /// - [Obsolete(locationObsolete)] - public static void ScrubLinesContaining(string extension, ScrubberLocation location, params string[] stringToMatch) => - ScrubLinesContaining(extension, StringComparison.OrdinalIgnoreCase, stringToMatch); - /// /// Remove the from the test results. /// public static void ScrubMachineName(string extension) => AddScrubber(extension, UserMachineScrubber.MachineScrubber()); - /// - /// Remove the from the test results. - /// - [Obsolete(locationObsolete)] - public static void ScrubMachineName(string extension, ScrubberLocation location) => - ScrubMachineName(extension); - /// /// Remove the from the test results. /// public static void ScrubUserName(string extension) => AddScrubber(extension, UserMachineScrubber.UserScrubber()); - - /// - /// Remove the from the test results. - /// - [Obsolete(locationObsolete)] - public static void ScrubUserName(string extension, ScrubberLocation location) => - ScrubUserName(extension); } diff --git a/src/Verify/Serialization/Scrubbers/VerifierSettings_ExtensionMappedGlobalScrubbers_Obsoletes.cs b/src/Verify/Serialization/Scrubbers/VerifierSettings_ExtensionMappedGlobalScrubbers_Obsoletes.cs new file mode 100644 index 0000000000..251bcfef35 --- /dev/null +++ b/src/Verify/Serialization/Scrubbers/VerifierSettings_ExtensionMappedGlobalScrubbers_Obsoletes.cs @@ -0,0 +1,61 @@ +namespace VerifyTests; + +public static partial class VerifierSettings +{ + /// + /// Remove any lines containing any of from the test results. + /// + [Obsolete(locationObsolete)] + public static void ScrubLinesContaining(string extension, StringComparison comparison, ScrubberLocation location, params string[] stringToMatch) => + ScrubLinesContaining(extension, comparison, stringToMatch); + + /// + /// Remove any lines matching from the test results. + /// + [Obsolete(locationObsolete)] + public static void ScrubLines(string extension, Func removeLine, ScrubberLocation location) => + ScrubLines(extension, removeLine); + + /// + /// Remove any lines containing only whitespace from the test results. + /// + [Obsolete(locationObsolete)] + public static void ScrubEmptyLines(string extension, ScrubberLocation location) => + ScrubEmptyLines(extension); + + /// + /// Replace inline s with a placeholder. + /// + [Obsolete(locationObsolete)] + public static void ScrubInlineGuids(string extension, ScrubberLocation location) => + ScrubInlineGuids(extension); + + /// + /// Scrub lines with an optional replace. + /// can return the input to ignore the line, or return a different string to replace it. + /// + [Obsolete(locationObsolete)] + public static void ScrubLinesWithReplace(string extension, Func replaceLine, ScrubberLocation location) => + ScrubLinesWithReplace(extension, replaceLine); + + /// + /// Remove any lines containing any of from the test results. + /// + [Obsolete(locationObsolete)] + public static void ScrubLinesContaining(string extension, ScrubberLocation location, params string[] stringToMatch) => + ScrubLinesContaining(extension, StringComparison.OrdinalIgnoreCase, stringToMatch); + + /// + /// Remove the from the test results. + /// + [Obsolete(locationObsolete)] + public static void ScrubMachineName(string extension, ScrubberLocation location) => + ScrubMachineName(extension); + + /// + /// Remove the from the test results. + /// + [Obsolete(locationObsolete)] + public static void ScrubUserName(string extension, ScrubberLocation location) => + ScrubUserName(extension); +} diff --git a/src/Verify/Serialization/Scrubbers/VerifierSettings_GlobalScrubbers.cs b/src/Verify/Serialization/Scrubbers/VerifierSettings_GlobalScrubbers.cs index 0df8925f6d..33abc4b1d0 100644 --- a/src/Verify/Serialization/Scrubbers/VerifierSettings_GlobalScrubbers.cs +++ b/src/Verify/Serialization/Scrubbers/VerifierSettings_GlobalScrubbers.cs @@ -6,8 +6,6 @@ public static partial class VerifierSettings internal static List GlobalSpanScrubbers = []; - const string locationObsolete = "ScrubberLocation is ignored; span scrubber ordering is engine determined. Use the overload without ScrubberLocation."; - /// /// Add a that applies to all verified files. /// @@ -54,13 +52,6 @@ public static void AddScrubber(Action AddScrubber(Scrubber.RemoveLinesContaining(comparison, stringToMatch)); - /// - /// Remove any lines containing any of from the test results. - /// - [Obsolete(locationObsolete)] - public static void ScrubLinesContaining(StringComparison comparison, ScrubberLocation location, params string[] stringToMatch) => - ScrubLinesContaining(comparison, stringToMatch); - /// /// Remove any lines matching from the test results. /// @@ -77,26 +68,12 @@ public static void ScrubLines(Func removeLine) => public static void ScrubLines(LineMatch removeLine) => AddScrubber(Scrubber.RemoveLines(removeLine)); - /// - /// Remove any lines matching from the test results. - /// - [Obsolete(locationObsolete)] - public static void ScrubLines(Func removeLine, ScrubberLocation location) => - ScrubLines(removeLine); - /// /// Remove any lines containing only whitespace from the test results. /// public static void ScrubEmptyLines() => AddScrubber(Scrubber.RemoveEmptyLines()); - /// - /// Remove any lines containing only whitespace from the test results. - /// - [Obsolete(locationObsolete)] - public static void ScrubEmptyLines(ScrubberLocation location) => - ScrubEmptyLines(); - internal static bool DateCountingEnabled { get; private set; } = true; /// @@ -118,16 +95,6 @@ public static void ScrubInlineDateTimes( } } - /// - /// Replace inline s with a placeholder. - /// - [Obsolete(locationObsolete)] - public static void ScrubInlineDateTimes( - [StringSyntax(StringSyntaxAttribute.DateTimeFormat)] string format, - Culture? culture, - ScrubberLocation location) => - ScrubInlineDateTimes(format, culture); - /// /// Replace inline s with a placeholder. /// @@ -141,16 +108,6 @@ public static void ScrubInlineDateTimeOffsets( } } - /// - /// Replace inline s with a placeholder. - /// - [Obsolete(locationObsolete)] - public static void ScrubInlineDateTimeOffsets( - [StringSyntax(StringSyntaxAttribute.DateTimeFormat)] string format, - Culture? culture, - ScrubberLocation location) => - ScrubInlineDateTimeOffsets(format, culture); - #if NET6_0_OR_GREATER /// @@ -166,16 +123,6 @@ public static void ScrubInlineDates( } } - /// - /// Replace inline s with a placeholder. - /// - [Obsolete(locationObsolete)] - public static void ScrubInlineDates( - [StringSyntax(StringSyntaxAttribute.DateOnlyFormat)] string format, - Culture? culture, - ScrubberLocation location) => - ScrubInlineDates(format, culture); - #endif /// @@ -184,13 +131,6 @@ public static void ScrubInlineDates( public static void ScrubInlineGuids() => AddScrubber(GuidMatcher.Instance); - /// - /// Replace inline s with a placeholder. - /// - [Obsolete(locationObsolete)] - public static void ScrubInlineGuids(ScrubberLocation location) => - ScrubInlineGuids(); - /// /// Scrub lines with an optional replace. /// can return the input to ignore the line, or return a different string to replace it. @@ -208,50 +148,21 @@ public static void ScrubLinesWithReplace(Func replaceLine) => public static void ScrubLinesWithReplace(LineReplace replaceLine) => AddScrubber(Scrubber.ReplaceLines(replaceLine)); - /// - /// Scrub lines with an optional replace. - /// can return the input to ignore the line, or return a different string to replace it. - /// - [Obsolete(locationObsolete)] - public static void ScrubLinesWithReplace(Func replaceLine, ScrubberLocation location) => - ScrubLinesWithReplace(replaceLine); - /// /// Remove any lines containing any of from the test results. /// public static void ScrubLinesContaining(params string[] stringToMatch) => ScrubLinesContaining(StringComparison.OrdinalIgnoreCase, stringToMatch); - /// - /// Remove any lines containing any of from the test results. - /// - [Obsolete(locationObsolete)] - public static void ScrubLinesContaining(ScrubberLocation location, params string[] stringToMatch) => - ScrubLinesContaining(StringComparison.OrdinalIgnoreCase, stringToMatch); - /// /// Remove the from the test results. /// public static void ScrubMachineName() => AddScrubber(UserMachineScrubber.MachineScrubber()); - /// - /// Remove the from the test results. - /// - [Obsolete(locationObsolete)] - public static void ScrubMachineName(ScrubberLocation location) => - ScrubMachineName(); - /// /// Remove the from the test results. /// public static void ScrubUserName() => AddScrubber(UserMachineScrubber.UserScrubber()); - - /// - /// Remove the from the test results. - /// - [Obsolete(locationObsolete)] - public static void ScrubUserName(ScrubberLocation location) => - ScrubUserName(); } diff --git a/src/Verify/Serialization/Scrubbers/VerifierSettings_GlobalScrubbers_Obsoletes.cs b/src/Verify/Serialization/Scrubbers/VerifierSettings_GlobalScrubbers_Obsoletes.cs new file mode 100644 index 0000000000..d842492206 --- /dev/null +++ b/src/Verify/Serialization/Scrubbers/VerifierSettings_GlobalScrubbers_Obsoletes.cs @@ -0,0 +1,97 @@ +namespace VerifyTests; + +public static partial class VerifierSettings +{ + const string locationObsolete = "ScrubberLocation is ignored; span scrubber ordering is engine determined. Use the overload without ScrubberLocation."; + + /// + /// Remove any lines containing any of from the test results. + /// + [Obsolete(locationObsolete)] + public static void ScrubLinesContaining(StringComparison comparison, ScrubberLocation location, params string[] stringToMatch) => + ScrubLinesContaining(comparison, stringToMatch); + + /// + /// Remove any lines matching from the test results. + /// + [Obsolete(locationObsolete)] + public static void ScrubLines(Func removeLine, ScrubberLocation location) => + ScrubLines(removeLine); + + /// + /// Remove any lines containing only whitespace from the test results. + /// + [Obsolete(locationObsolete)] + public static void ScrubEmptyLines(ScrubberLocation location) => + ScrubEmptyLines(); + + /// + /// Replace inline s with a placeholder. + /// + [Obsolete(locationObsolete)] + public static void ScrubInlineDateTimes( + [StringSyntax(StringSyntaxAttribute.DateTimeFormat)] string format, + Culture? culture, + ScrubberLocation location) => + ScrubInlineDateTimes(format, culture); + + /// + /// Replace inline s with a placeholder. + /// + [Obsolete(locationObsolete)] + public static void ScrubInlineDateTimeOffsets( + [StringSyntax(StringSyntaxAttribute.DateTimeFormat)] string format, + Culture? culture, + ScrubberLocation location) => + ScrubInlineDateTimeOffsets(format, culture); + + /// + /// Replace inline s with a placeholder. + /// + [Obsolete(locationObsolete)] + public static void ScrubInlineGuids(ScrubberLocation location) => + ScrubInlineGuids(); + + /// + /// Scrub lines with an optional replace. + /// can return the input to ignore the line, or return a different string to replace it. + /// + [Obsolete(locationObsolete)] + public static void ScrubLinesWithReplace(Func replaceLine, ScrubberLocation location) => + ScrubLinesWithReplace(replaceLine); + + /// + /// Remove any lines containing any of from the test results. + /// + [Obsolete(locationObsolete)] + public static void ScrubLinesContaining(ScrubberLocation location, params string[] stringToMatch) => + ScrubLinesContaining(StringComparison.OrdinalIgnoreCase, stringToMatch); + + /// + /// Remove the from the test results. + /// + [Obsolete(locationObsolete)] + public static void ScrubMachineName(ScrubberLocation location) => + ScrubMachineName(); + + /// + /// Remove the from the test results. + /// + [Obsolete(locationObsolete)] + public static void ScrubUserName(ScrubberLocation location) => + ScrubUserName(); + +#if NET6_0_OR_GREATER + + /// + /// Replace inline s with a placeholder. + /// + [Obsolete(locationObsolete)] + public static void ScrubInlineDates( + [StringSyntax(StringSyntaxAttribute.DateOnlyFormat)] string format, + Culture? culture, + ScrubberLocation location) => + ScrubInlineDates(format, culture); + +#endif +} diff --git a/src/Verify/Serialization/Scrubbers/VerifySettings_ExtensionMappedInstanceScrubbers.cs b/src/Verify/Serialization/Scrubbers/VerifySettings_ExtensionMappedInstanceScrubbers.cs index ac4f834b90..5d79f0000c 100644 --- a/src/Verify/Serialization/Scrubbers/VerifySettings_ExtensionMappedInstanceScrubbers.cs +++ b/src/Verify/Serialization/Scrubbers/VerifySettings_ExtensionMappedInstanceScrubbers.cs @@ -64,52 +64,24 @@ public void AddScrubber(string extension, Action AddScrubber(extension, UserMachineScrubber.MachineScrubber()); - /// - /// Remove the from the test results. - /// - [Obsolete(locationObsolete)] - public void ScrubMachineName(string extension, ScrubberLocation location) => - ScrubMachineName(extension); - /// /// Remove the from the test results. /// public void ScrubUserName(string extension) => AddScrubber(extension, UserMachineScrubber.UserScrubber()); - /// - /// Remove the from the test results. - /// - [Obsolete(locationObsolete)] - public void ScrubUserName(string extension, ScrubberLocation location) => - ScrubUserName(extension); - /// /// Remove any lines containing any of from the test results. /// public void ScrubLinesContaining(string extension, StringComparison comparison, params string[] stringToMatch) => AddScrubber(extension, Scrubber.RemoveLinesContaining(comparison, stringToMatch)); - /// - /// Remove any lines containing any of from the test results. - /// - [Obsolete(locationObsolete)] - public void ScrubLinesContaining(string extension, StringComparison comparison, ScrubberLocation location, params string[] stringToMatch) => - ScrubLinesContaining(extension, comparison, stringToMatch); - /// /// Replace inline s with a placeholder. /// public void ScrubInlineGuids(string extension) => AddScrubber(extension, GuidMatcher.Instance); - /// - /// Replace inline s with a placeholder. - /// - [Obsolete(locationObsolete)] - public void ScrubInlineGuids(string extension, ScrubberLocation location) => - ScrubInlineGuids(extension); - /// /// Remove any lines matching from the test results. /// @@ -126,13 +98,6 @@ public void ScrubLines(string extension, Func removeLine) => public void ScrubLines(string extension, LineMatch removeLine) => AddScrubber(extension, Scrubber.RemoveLines(removeLine)); - /// - /// Remove any lines matching from the test results. - /// - [Obsolete(locationObsolete)] - public void ScrubLines(string extension, Func removeLine, ScrubberLocation location) => - ScrubLines(extension, removeLine); - /// /// Scrub lines with an optional replace. /// can return the input to ignore the line, or return a different string to replace it. @@ -150,31 +115,9 @@ public void ScrubLinesWithReplace(string extension, Func replac public void ScrubLinesWithReplace(string extension, LineReplace replaceLine) => AddScrubber(extension, Scrubber.ReplaceLines(replaceLine)); - /// - /// Scrub lines with an optional replace. - /// can return the input to ignore the line, or return a different string to replace it. - /// - [Obsolete(locationObsolete)] - public void ScrubLinesWithReplace(string extension, Func replaceLine, ScrubberLocation location) => - ScrubLinesWithReplace(extension, replaceLine); - /// /// Remove any lines containing only whitespace from the test results. /// public void ScrubEmptyLines(string extension) => AddScrubber(extension, Scrubber.RemoveEmptyLines()); - - /// - /// Remove any lines containing only whitespace from the test results. - /// - [Obsolete(locationObsolete)] - public void ScrubEmptyLines(string extension, ScrubberLocation location) => - ScrubEmptyLines(extension); - - /// - /// Remove any lines containing any of from the test results. - /// - [Obsolete(locationObsolete)] - public void ScrubLinesContaining(string extension, ScrubberLocation location, params string[] stringToMatch) => - ScrubLinesContaining(extension, StringComparison.OrdinalIgnoreCase, stringToMatch); } diff --git a/src/Verify/Serialization/Scrubbers/VerifySettings_ExtensionMappedInstanceScrubbers_Obsoletes.cs b/src/Verify/Serialization/Scrubbers/VerifySettings_ExtensionMappedInstanceScrubbers_Obsoletes.cs new file mode 100644 index 0000000000..5c3fc07dec --- /dev/null +++ b/src/Verify/Serialization/Scrubbers/VerifySettings_ExtensionMappedInstanceScrubbers_Obsoletes.cs @@ -0,0 +1,61 @@ +namespace VerifyTests; + +public partial class VerifySettings +{ + /// + /// Remove the from the test results. + /// + [Obsolete(locationObsolete)] + public void ScrubMachineName(string extension, ScrubberLocation location) => + ScrubMachineName(extension); + + /// + /// Remove the from the test results. + /// + [Obsolete(locationObsolete)] + public void ScrubUserName(string extension, ScrubberLocation location) => + ScrubUserName(extension); + + /// + /// Remove any lines containing any of from the test results. + /// + [Obsolete(locationObsolete)] + public void ScrubLinesContaining(string extension, StringComparison comparison, ScrubberLocation location, params string[] stringToMatch) => + ScrubLinesContaining(extension, comparison, stringToMatch); + + /// + /// Replace inline s with a placeholder. + /// + [Obsolete(locationObsolete)] + public void ScrubInlineGuids(string extension, ScrubberLocation location) => + ScrubInlineGuids(extension); + + /// + /// Remove any lines matching from the test results. + /// + [Obsolete(locationObsolete)] + public void ScrubLines(string extension, Func removeLine, ScrubberLocation location) => + ScrubLines(extension, removeLine); + + /// + /// Scrub lines with an optional replace. + /// can return the input to ignore the line, or return a different string to replace it. + /// + [Obsolete(locationObsolete)] + public void ScrubLinesWithReplace(string extension, Func replaceLine, ScrubberLocation location) => + ScrubLinesWithReplace(extension, replaceLine); + + /// + /// Remove any lines containing only whitespace from the test results. + /// + [Obsolete(locationObsolete)] + public void ScrubEmptyLines(string extension, ScrubberLocation location) => + ScrubEmptyLines(extension); + + /// + /// Remove any lines containing any of from the test results. + /// + [Obsolete(locationObsolete)] + public void ScrubLinesContaining(string extension, ScrubberLocation location, params string[] stringToMatch) => + ScrubLinesContaining(extension, StringComparison.OrdinalIgnoreCase, stringToMatch); +} diff --git a/src/Verify/Serialization/Scrubbers/VerifySettings_InstanceScrubbers.cs b/src/Verify/Serialization/Scrubbers/VerifySettings_InstanceScrubbers.cs index 4c2505ed57..d78de599fb 100644 --- a/src/Verify/Serialization/Scrubbers/VerifySettings_InstanceScrubbers.cs +++ b/src/Verify/Serialization/Scrubbers/VerifySettings_InstanceScrubbers.cs @@ -10,8 +10,6 @@ public partial class VerifySettings // serialized string value. Invalidated on registration. internal EngineScrubberSet? PropertyValueSetCache; - const string locationObsolete = "ScrubberLocation is ignored; span scrubber ordering is engine determined. Use the overload without ScrubberLocation."; - /// /// Add a . /// @@ -36,26 +34,12 @@ public void AddScrubber(Scrubber scrubber) public void ScrubMachineName() => AddScrubber(UserMachineScrubber.MachineScrubber()); - /// - /// Remove the from the test results. - /// - [Obsolete(locationObsolete)] - public void ScrubMachineName(ScrubberLocation location) => - ScrubMachineName(); - /// /// Remove the from the test results. /// public void ScrubUserName() => AddScrubber(UserMachineScrubber.UserScrubber()); - /// - /// Remove the from the test results. - /// - [Obsolete(locationObsolete)] - public void ScrubUserName(ScrubberLocation location) => - ScrubUserName(); - /// /// Modify the resulting test content using custom code. /// @@ -99,13 +83,6 @@ public void AddScrubber(Action AddScrubber(Scrubber.RemoveLinesContaining(comparison, stringToMatch)); - /// - /// Remove any lines containing any of from the test results. - /// - [Obsolete(locationObsolete)] - public void ScrubLinesContaining(StringComparison comparison, ScrubberLocation location, params string[] stringToMatch) => - ScrubLinesContaining(comparison, stringToMatch); - /// /// Replace inline s with a placeholder. /// @@ -119,13 +96,6 @@ public void ScrubInlineGuids() AddScrubber(GuidMatcher.Instance); } - /// - /// Replace inline s with a placeholder. - /// - [Obsolete(locationObsolete)] - public void ScrubInlineGuids(ScrubberLocation location) => - ScrubInlineGuids(); - /// /// Replace inline s with a placeholder. /// @@ -145,17 +115,6 @@ public void ScrubInlineDateTimes( } } - /// - /// Replace inline s with a placeholder. - /// - [Obsolete(locationObsolete)] - public void ScrubInlineDateTimes( - [StringSyntax(StringSyntaxAttribute.DateTimeFormat)] - string format, - Culture? culture, - ScrubberLocation location) => - ScrubInlineDateTimes(format, culture); - /// /// Replace inline s with a placeholder. /// @@ -175,17 +134,6 @@ public void ScrubInlineDateTimeOffsets( } } - /// - /// Replace inline s with a placeholder. - /// - [Obsolete(locationObsolete)] - public void ScrubInlineDateTimeOffsets( - [StringSyntax(StringSyntaxAttribute.DateTimeFormat)] - string format, - Culture? culture, - ScrubberLocation location) => - ScrubInlineDateTimeOffsets(format, culture); - #if NET6_0_OR_GREATER /// @@ -206,16 +154,6 @@ public void ScrubInlineDates( } } - /// - /// Replace inline s with a placeholder. - /// - [Obsolete(locationObsolete)] - public void ScrubInlineDates( - [StringSyntax(StringSyntaxAttribute.DateOnlyFormat)] string format, - Culture? culture, - ScrubberLocation location) => - ScrubInlineDates(format, culture); - #endif /// @@ -234,13 +172,6 @@ public void ScrubLines(Func removeLine) => public void ScrubLines(LineMatch removeLine) => AddScrubber(Scrubber.RemoveLines(removeLine)); - /// - /// Remove any lines matching from the test results. - /// - [Obsolete(locationObsolete)] - public void ScrubLines(Func removeLine, ScrubberLocation location) => - ScrubLines(removeLine); - /// /// Scrub lines with an optional replace. /// can return the input to ignore the line, or return a different string to replace it. @@ -258,37 +189,15 @@ public void ScrubLinesWithReplace(Func replaceLine) => public void ScrubLinesWithReplace(LineReplace replaceLine) => AddScrubber(Scrubber.ReplaceLines(replaceLine)); - /// - /// Scrub lines with an optional replace. - /// can return the input to ignore the line, or return a different string to replace it. - /// - [Obsolete(locationObsolete)] - public void ScrubLinesWithReplace(Func replaceLine, ScrubberLocation location) => - ScrubLinesWithReplace(replaceLine); - /// /// Remove any lines containing only whitespace from the test results. /// public void ScrubEmptyLines() => AddScrubber(Scrubber.RemoveEmptyLines()); - /// - /// Remove any lines containing only whitespace from the test results. - /// - [Obsolete(locationObsolete)] - public void ScrubEmptyLines(ScrubberLocation location) => - ScrubEmptyLines(); - /// /// Remove any lines containing any of from the test results. /// public void ScrubLinesContaining(params string[] stringToMatch) => ScrubLinesContaining(StringComparison.OrdinalIgnoreCase, stringToMatch); - - /// - /// Remove any lines containing any of from the test results. - /// - [Obsolete(locationObsolete)] - public void ScrubLinesContaining(ScrubberLocation location, params string[] stringToMatch) => - ScrubLinesContaining(StringComparison.OrdinalIgnoreCase, stringToMatch); } diff --git a/src/Verify/Serialization/Scrubbers/VerifySettings_InstanceScrubbers_Obsoletes.cs b/src/Verify/Serialization/Scrubbers/VerifySettings_InstanceScrubbers_Obsoletes.cs new file mode 100644 index 0000000000..f37870534e --- /dev/null +++ b/src/Verify/Serialization/Scrubbers/VerifySettings_InstanceScrubbers_Obsoletes.cs @@ -0,0 +1,99 @@ +namespace VerifyTests; + +public partial class VerifySettings +{ + const string locationObsolete = "ScrubberLocation is ignored; span scrubber ordering is engine determined. Use the overload without ScrubberLocation."; + + /// + /// Remove the from the test results. + /// + [Obsolete(locationObsolete)] + public void ScrubMachineName(ScrubberLocation location) => + ScrubMachineName(); + + /// + /// Remove the from the test results. + /// + [Obsolete(locationObsolete)] + public void ScrubUserName(ScrubberLocation location) => + ScrubUserName(); + + /// + /// Remove any lines containing any of from the test results. + /// + [Obsolete(locationObsolete)] + public void ScrubLinesContaining(StringComparison comparison, ScrubberLocation location, params string[] stringToMatch) => + ScrubLinesContaining(comparison, stringToMatch); + + /// + /// Replace inline s with a placeholder. + /// + [Obsolete(locationObsolete)] + public void ScrubInlineGuids(ScrubberLocation location) => + ScrubInlineGuids(); + + /// + /// Replace inline s with a placeholder. + /// + [Obsolete(locationObsolete)] + public void ScrubInlineDateTimes( + [StringSyntax(StringSyntaxAttribute.DateTimeFormat)] + string format, + Culture? culture, + ScrubberLocation location) => + ScrubInlineDateTimes(format, culture); + + /// + /// Replace inline s with a placeholder. + /// + [Obsolete(locationObsolete)] + public void ScrubInlineDateTimeOffsets( + [StringSyntax(StringSyntaxAttribute.DateTimeFormat)] + string format, + Culture? culture, + ScrubberLocation location) => + ScrubInlineDateTimeOffsets(format, culture); + + /// + /// Remove any lines matching from the test results. + /// + [Obsolete(locationObsolete)] + public void ScrubLines(Func removeLine, ScrubberLocation location) => + ScrubLines(removeLine); + + /// + /// Scrub lines with an optional replace. + /// can return the input to ignore the line, or return a different string to replace it. + /// + [Obsolete(locationObsolete)] + public void ScrubLinesWithReplace(Func replaceLine, ScrubberLocation location) => + ScrubLinesWithReplace(replaceLine); + + /// + /// Remove any lines containing only whitespace from the test results. + /// + [Obsolete(locationObsolete)] + public void ScrubEmptyLines(ScrubberLocation location) => + ScrubEmptyLines(); + + /// + /// Remove any lines containing any of from the test results. + /// + [Obsolete(locationObsolete)] + public void ScrubLinesContaining(ScrubberLocation location, params string[] stringToMatch) => + ScrubLinesContaining(StringComparison.OrdinalIgnoreCase, stringToMatch); + +#if NET6_0_OR_GREATER + + /// + /// Replace inline s with a placeholder. + /// + [Obsolete(locationObsolete)] + public void ScrubInlineDates( + [StringSyntax(StringSyntaxAttribute.DateOnlyFormat)] string format, + Culture? culture, + ScrubberLocation location) => + ScrubInlineDates(format, culture); + +#endif +} diff --git a/src/Verify/SettingsTask_Scrubbing.cs b/src/Verify/SettingsTask_Scrubbing.cs index 3558e24970..7fc28525fa 100644 --- a/src/Verify/SettingsTask_Scrubbing.cs +++ b/src/Verify/SettingsTask_Scrubbing.cs @@ -2,8 +2,6 @@ namespace VerifyTests; public partial class SettingsTask { - const string locationObsolete = "ScrubberLocation is ignored; span scrubber ordering is engine determined. Use the overload without ScrubberLocation."; - /// [Pure] public SettingsTask DisableScrubbers() @@ -52,12 +50,6 @@ public SettingsTask ScrubInlineGuids() return this; } - /// - [Obsolete(locationObsolete)] - [Pure] - public SettingsTask ScrubInlineGuids(ScrubberLocation location) => - ScrubInlineGuids(); - /// [Pure] public SettingsTask ScrubGuids() @@ -82,12 +74,6 @@ public SettingsTask ScrubInlineGuids(string extension) return this; } - /// - [Obsolete(locationObsolete)] - [Pure] - public SettingsTask ScrubInlineGuids(string extension, ScrubberLocation location) => - ScrubInlineGuids(extension); - /// [Pure] public SettingsTask DisableDateCounting() @@ -114,15 +100,6 @@ public SettingsTask ScrubInlineDateTimes( return this; } - /// - [Obsolete(locationObsolete)] - [Pure] - public SettingsTask ScrubInlineDateTimes( - [StringSyntax(StringSyntaxAttribute.DateTimeFormat)] string format, - Culture? culture, - ScrubberLocation location) => - ScrubInlineDateTimes(format, culture); - /// [Pure] public SettingsTask ScrubInlineDateTimeOffsets( @@ -133,15 +110,6 @@ public SettingsTask ScrubInlineDateTimeOffsets( return this; } - /// - [Obsolete(locationObsolete)] - [Pure] - public SettingsTask ScrubInlineDateTimeOffsets( - [StringSyntax(StringSyntaxAttribute.DateTimeFormat)] string format, - Culture? culture, - ScrubberLocation location) => - ScrubInlineDateTimeOffsets(format, culture); - #if NET6_0_OR_GREATER /// @@ -154,15 +122,6 @@ public SettingsTask ScrubInlineDates( return this; } - /// - [Obsolete(locationObsolete)] - [Pure] - public SettingsTask ScrubInlineDates( - [StringSyntax(StringSyntaxAttribute.DateOnlyFormat)] string format, - Culture? culture, - ScrubberLocation location) => - ScrubInlineDates(format, culture); - #endif /// @@ -173,12 +132,6 @@ public SettingsTask ScrubMachineName() return this; } - /// - [Obsolete(locationObsolete)] - [Pure] - public SettingsTask ScrubMachineName(ScrubberLocation location) => - ScrubMachineName(); - /// [Pure] public SettingsTask ScrubMachineName(string extension) @@ -187,12 +140,6 @@ public SettingsTask ScrubMachineName(string extension) return this; } - /// - [Obsolete(locationObsolete)] - [Pure] - public SettingsTask ScrubMachineName(string extension, ScrubberLocation location) => - ScrubMachineName(extension); - /// [Pure] public SettingsTask ScrubUserName() @@ -201,12 +148,6 @@ public SettingsTask ScrubUserName() return this; } - /// - [Obsolete(locationObsolete)] - [Pure] - public SettingsTask ScrubUserName(ScrubberLocation location) => - ScrubUserName(); - /// [Pure] public SettingsTask ScrubUserName(string extension) @@ -215,12 +156,6 @@ public SettingsTask ScrubUserName(string extension) return this; } - /// - [Obsolete(locationObsolete)] - [Pure] - public SettingsTask ScrubUserName(string extension, ScrubberLocation location) => - ScrubUserName(extension); - /// [Pure] public SettingsTask ScrubLinesContaining(StringComparison comparison, params string[] stringToMatch) @@ -237,18 +172,6 @@ public SettingsTask ScrubLinesContaining(string extension, StringComparison comp return this; } - /// - [Obsolete(locationObsolete)] - [Pure] - public SettingsTask ScrubLinesContaining(StringComparison comparison, ScrubberLocation location, params string[] stringToMatch) => - ScrubLinesContaining(comparison, stringToMatch); - - /// - [Obsolete(locationObsolete)] - [Pure] - public SettingsTask ScrubLinesContaining(string extension, StringComparison comparison, ScrubberLocation location, params string[] stringToMatch) => - ScrubLinesContaining(extension, comparison, stringToMatch); - /// [Pure] public SettingsTask ScrubLines(Func removeLine) @@ -266,12 +189,6 @@ public SettingsTask ScrubLines(LineMatch removeLine) return this; } - /// - [Obsolete(locationObsolete)] - [Pure] - public SettingsTask ScrubLines(Func removeLine, ScrubberLocation location) => - ScrubLines(removeLine); - /// [Pure] public SettingsTask ScrubLines(string extension, Func removeLine) @@ -280,21 +197,6 @@ public SettingsTask ScrubLines(string extension, Func removeLine) return this; } - /// - [OverloadResolutionPriority(-1)] - [Pure] - public SettingsTask ScrubLines(string extension, LineMatch removeLine) - { - CurrentSettings.ScrubLines(extension, removeLine); - return this; - } - - /// - [Obsolete(locationObsolete)] - [Pure] - public SettingsTask ScrubLines(string extension, Func removeLine, ScrubberLocation location) => - ScrubLines(extension, removeLine); - /// [Pure] public SettingsTask ScrubLinesWithReplace(Func replaceLine) @@ -312,12 +214,6 @@ public SettingsTask ScrubLinesWithReplace(LineReplace replaceLine) return this; } - /// - [Obsolete(locationObsolete)] - [Pure] - public SettingsTask ScrubLinesWithReplace(Func replaceLine, ScrubberLocation location) => - ScrubLinesWithReplace(replaceLine); - /// [Pure] public SettingsTask ScrubLinesWithReplace(string extension, Func replaceLine) @@ -335,12 +231,6 @@ public SettingsTask ScrubLinesWithReplace(string extension, LineReplace replaceL return this; } - /// - [Obsolete(locationObsolete)] - [Pure] - public SettingsTask ScrubLinesWithReplace(string extension, Func replaceLine, ScrubberLocation location) => - ScrubLinesWithReplace(extension, replaceLine); - /// [Pure] public SettingsTask ScrubEmptyLines() @@ -349,12 +239,6 @@ public SettingsTask ScrubEmptyLines() return this; } - /// - [Obsolete(locationObsolete)] - [Pure] - public SettingsTask ScrubEmptyLines(ScrubberLocation location) => - ScrubEmptyLines(); - /// [Pure] public SettingsTask ScrubEmptyLines(string extension) @@ -363,12 +247,6 @@ public SettingsTask ScrubEmptyLines(string extension) return this; } - /// - [Obsolete(locationObsolete)] - [Pure] - public SettingsTask ScrubEmptyLines(string extension, ScrubberLocation location) => - ScrubEmptyLines(extension); - /// [Pure] public SettingsTask ScrubLinesContaining(params string[] stringToMatch) @@ -376,19 +254,4 @@ public SettingsTask ScrubLinesContaining(params string[] stringToMatch) CurrentSettings.ScrubLinesContaining(stringToMatch); return this; } - - /// - [Obsolete(locationObsolete)] - [Pure] - public SettingsTask ScrubLinesContaining(ScrubberLocation location, params string[] stringToMatch) => - ScrubLinesContaining(stringToMatch); - - /// - [Obsolete(locationObsolete)] - [Pure] - public SettingsTask ScrubLinesContaining(string extension, ScrubberLocation location, params string[] stringToMatch) - { - CurrentSettings.ScrubLinesContaining(extension, StringComparison.OrdinalIgnoreCase, stringToMatch); - return this; - } } diff --git a/src/Verify/SettingsTask_Scrubbing_Obsoletes.cs b/src/Verify/SettingsTask_Scrubbing_Obsoletes.cs new file mode 100644 index 0000000000..c468cb0194 --- /dev/null +++ b/src/Verify/SettingsTask_Scrubbing_Obsoletes.cs @@ -0,0 +1,147 @@ +namespace VerifyTests; + +public partial class SettingsTask +{ + const string locationObsolete = "ScrubberLocation is ignored; span scrubber ordering is engine determined. Use the overload without ScrubberLocation."; + + /// + [Obsolete(locationObsolete)] + [Pure] + public SettingsTask ScrubInlineGuids(ScrubberLocation location) => + ScrubInlineGuids(); + + /// + [Obsolete(locationObsolete)] + [Pure] + public SettingsTask ScrubInlineGuids(string extension, ScrubberLocation location) => + ScrubInlineGuids(extension); + + /// + [Obsolete(locationObsolete)] + [Pure] + public SettingsTask ScrubInlineDateTimes( + [StringSyntax(StringSyntaxAttribute.DateTimeFormat)] + string format, + Culture? culture, + ScrubberLocation location) => + ScrubInlineDateTimes(format, culture); + + /// + [Obsolete(locationObsolete)] + [Pure] + public SettingsTask ScrubInlineDateTimeOffsets( + [StringSyntax(StringSyntaxAttribute.DateTimeFormat)] + string format, + Culture? culture, + ScrubberLocation location) => + ScrubInlineDateTimeOffsets(format, culture); + +#if NET6_0_OR_GREATER + + /// + [Obsolete(locationObsolete)] + [Pure] + public SettingsTask ScrubInlineDates( + [StringSyntax(StringSyntaxAttribute.DateOnlyFormat)] string format, + Culture? culture, + ScrubberLocation location) => + ScrubInlineDates(format, culture); + +#endif + + /// + [Obsolete(locationObsolete)] + [Pure] + public SettingsTask ScrubMachineName(ScrubberLocation location) => + ScrubMachineName(); + + /// + [Obsolete(locationObsolete)] + [Pure] + public SettingsTask ScrubMachineName(string extension, ScrubberLocation location) => + ScrubMachineName(extension); + + /// + [Obsolete(locationObsolete)] + [Pure] + public SettingsTask ScrubUserName(ScrubberLocation location) => + ScrubUserName(); + + /// + [Obsolete(locationObsolete)] + [Pure] + public SettingsTask ScrubUserName(string extension, ScrubberLocation location) => + ScrubUserName(extension); + + /// + [Obsolete(locationObsolete)] + [Pure] + public SettingsTask ScrubLinesContaining(StringComparison comparison, ScrubberLocation location, params string[] stringToMatch) => + ScrubLinesContaining(comparison, stringToMatch); + + /// + [Obsolete(locationObsolete)] + [Pure] + public SettingsTask ScrubLinesContaining(string extension, StringComparison comparison, ScrubberLocation location, params string[] stringToMatch) => + ScrubLinesContaining(extension, comparison, stringToMatch); + + /// + [Obsolete(locationObsolete)] + [Pure] + public SettingsTask ScrubLines(Func removeLine, ScrubberLocation location) => + ScrubLines(removeLine); + + /// + [OverloadResolutionPriority(-1)] + [Pure] + public SettingsTask ScrubLines(string extension, LineMatch removeLine) + { + CurrentSettings.ScrubLines(extension, removeLine); + return this; + } + + /// + [Obsolete(locationObsolete)] + [Pure] + public SettingsTask ScrubLines(string extension, Func removeLine, ScrubberLocation location) => + ScrubLines(extension, removeLine); + + /// + [Obsolete(locationObsolete)] + [Pure] + public SettingsTask ScrubLinesWithReplace(Func replaceLine, ScrubberLocation location) => + ScrubLinesWithReplace(replaceLine); + + /// + [Obsolete(locationObsolete)] + [Pure] + public SettingsTask ScrubLinesWithReplace(string extension, Func replaceLine, ScrubberLocation location) => + ScrubLinesWithReplace(extension, replaceLine); + + /// + [Obsolete(locationObsolete)] + [Pure] + public SettingsTask ScrubEmptyLines(ScrubberLocation location) => + ScrubEmptyLines(); + + /// + [Obsolete(locationObsolete)] + [Pure] + public SettingsTask ScrubEmptyLines(string extension, ScrubberLocation location) => + ScrubEmptyLines(extension); + + /// + [Obsolete(locationObsolete)] + [Pure] + public SettingsTask ScrubLinesContaining(ScrubberLocation location, params string[] stringToMatch) => + ScrubLinesContaining(stringToMatch); + + /// + [Obsolete(locationObsolete)] + [Pure] + public SettingsTask ScrubLinesContaining(string extension, ScrubberLocation location, params string[] stringToMatch) + { + CurrentSettings.ScrubLinesContaining(extension, StringComparison.OrdinalIgnoreCase, stringToMatch); + return this; + } +} From c62910b39ed9b09269e9588a3df875a100864126 Mon Sep 17 00:00:00 2001 From: Simon Cropp Date: Sat, 18 Jul 2026 22:56:11 +1000 Subject: [PATCH 15/33] Update appveyor.yml --- src/appveyor.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/src/appveyor.yml b/src/appveyor.yml index b8ae3c54c3..0162537672 100644 --- a/src/appveyor.yml +++ b/src/appveyor.yml @@ -28,6 +28,7 @@ before_build: - dotnet tool restore --tool-manifest src/.config/dotnet-tools.json build_script: - dotnet build src/Verify.slnx --configuration Release --verbosity minimal +- dotnet test src/ApplyScrubbersTests --configuration Release --no-build --no-restore --verbosity minimal - dotnet test src/DeterministicTests --configuration Release --no-build --no-restore --verbosity minimal - dotnet test src/FSharpTests --configuration Release --no-build --no-restore --verbosity minimal - dotnet test src/StaticSettingsTests --configuration Release --no-build --no-restore --verbosity minimal From afd962972ca8d0c8ce21af637389192f644df551 Mon Sep 17 00:00:00 2001 From: Simon Cropp Date: Sat, 18 Jul 2026 23:01:04 +1000 Subject: [PATCH 16/33] . --- docs/counter.md | 4 ++-- docs/mdsource/counter.source.md | 4 ++-- docs/mdsource/scrubbers.source.md | 2 +- docs/scrubbers.md | 2 +- release-notes-32.0.0.md | 8 ++++---- src/Verify/Serialization/Scrubbers/Scrubber.cs | 4 ++-- .../VerifierSettings_ExtensionMappedGlobalScrubbers.cs | 4 ++-- .../Scrubbers/VerifierSettings_GlobalScrubbers.cs | 4 ++-- .../VerifySettings_ExtensionMappedInstanceScrubbers.cs | 4 ++-- .../Scrubbers/VerifySettings_InstanceScrubbers.cs | 4 ++-- 10 files changed, 20 insertions(+), 20 deletions(-) diff --git a/docs/counter.md b/docs/counter.md index 303be31fc5..1070534106 100644 --- a/docs/counter.md +++ b/docs/counter.md @@ -7,7 +7,7 @@ To change this file edit the source file and then run MarkdownSnippets. # Counter -The `Counter` class provides methods to attempt conversion of a `CharSpan` value into a string representation of supported types. +The `Counter` class provides methods to attempt conversion of a `ReadOnlySpan` value into a string representation of supported types. Supported types include @@ -18,7 +18,7 @@ It handles matching equivalent values and assigning a number to each match. It i ## TryConvert -Takes a CharSpan and attempts to parse it to one of the supported types, then return the tokenized scrubbed value for that value. +Takes a `ReadOnlySpan` and attempts to parse it to one of the supported types, then return the tokenized scrubbed value for that value. One example usage is inside a custom scrubber: diff --git a/docs/mdsource/counter.source.md b/docs/mdsource/counter.source.md index 06854a0af2..2f7f3012ba 100644 --- a/docs/mdsource/counter.source.md +++ b/docs/mdsource/counter.source.md @@ -1,6 +1,6 @@ # Counter -The `Counter` class provides methods to attempt conversion of a `CharSpan` value into a string representation of supported types. +The `Counter` class provides methods to attempt conversion of a `ReadOnlySpan` value into a string representation of supported types. Supported types include @@ -11,7 +11,7 @@ It handles matching equivalent values and assigning a number to each match. It i ## TryConvert -Takes a CharSpan and attempts to parse it to one of the supported types, then return the tokenized scrubbed value for that value. +Takes a `ReadOnlySpan` and attempts to parse it to one of the supported types, then return the tokenized scrubbed value for that value. One example usage is inside a custom scrubber: diff --git a/docs/mdsource/scrubbers.source.md b/docs/mdsource/scrubbers.source.md index ac6147bf54..9518e3fca9 100644 --- a/docs/mdsource/scrubbers.source.md +++ b/docs/mdsource/scrubbers.source.md @@ -21,7 +21,7 @@ A `Scrubber` is created via static factory methods and registered via `AddScrubb snippet: AddScrubberEngine -`ScrubLines` and `ScrubLinesWithReplace` (and the corresponding `Scrubber` factories) also accept span based delegates (`LineMatch` / `LineReplace`) that avoid allocating a string per line. Use an explicitly typed lambda parameter to select them, e.g. `ScrubLines((CharSpan line) => ...)`; untyped lambdas bind the string overloads. +`ScrubLines` and `ScrubLinesWithReplace` (and the corresponding `Scrubber` factories) also accept span based delegates (`LineMatch` / `LineReplace`) that avoid allocating a string per line. Use an explicitly typed lambda parameter to select them, e.g. `ScrubLines((ReadOnlySpan line) => ...)`; untyped lambdas bind the string overloads. Engine semantics: diff --git a/docs/scrubbers.md b/docs/scrubbers.md index aff4eb7c43..0b3f019f75 100644 --- a/docs/scrubbers.md +++ b/docs/scrubbers.md @@ -48,7 +48,7 @@ verifySettings.AddScrubber( snippet source | anchor -`ScrubLines` and `ScrubLinesWithReplace` (and the corresponding `Scrubber` factories) also accept span based delegates (`LineMatch` / `LineReplace`) that avoid allocating a string per line. Use an explicitly typed lambda parameter to select them, e.g. `ScrubLines((CharSpan line) => ...)`; untyped lambdas bind the string overloads. +`ScrubLines` and `ScrubLinesWithReplace` (and the corresponding `Scrubber` factories) also accept span based delegates (`LineMatch` / `LineReplace`) that avoid allocating a string per line. Use an explicitly typed lambda parameter to select them, e.g. `ScrubLines((ReadOnlySpan line) => ...)`; untyped lambdas bind the string overloads. Engine semantics: diff --git a/release-notes-32.0.0.md b/release-notes-32.0.0.md index b4984717ff..8f39c50425 100644 --- a/release-notes-32.0.0.md +++ b/release-notes-32.0.0.md @@ -94,7 +94,7 @@ multi-line regex) should stay on the `AddScrubber(Action)` overlo configurations listed under "Behavior" above; a suite that does not rely on scrubber ordering or sugar-vs-legacy interaction should see no changes. 4. Optional: for hot custom line predicates, switch to the new span overloads to avoid a per-line string - allocation, for example `ScrubLines((CharSpan line) => ...)`. + allocation, for example `ScrubLines((ReadOnlySpan line) => ...)`. ## New public API @@ -114,12 +114,12 @@ settings.AddScrubber(Scrubber.Match((segment, counter, context, out index, out l // Line scoped settings.AddScrubber(Scrubber.RemoveLinesContaining("token")); settings.AddScrubber(Scrubber.RemoveEmptyLines()); -settings.AddScrubber(Scrubber.RemoveLines((CharSpan line) => ...)); // span predicate, no per-line alloc -settings.AddScrubber(Scrubber.ReplaceLines((CharSpan line) => ...)); // returns LineResult.Keep / Remove / Replace(text) +settings.AddScrubber(Scrubber.RemoveLines((ReadOnlySpan line) => ...)); // span predicate, no per-line alloc +settings.AddScrubber(Scrubber.ReplaceLines((ReadOnlySpan line) => ...)); // returns LineResult.Keep / Remove / Replace(text) ``` `ScrubLines` and `ScrubLinesWithReplace` also gained span-delegate overloads (`LineMatch` / -`LineReplace`); select them with an explicitly typed lambda parameter (`(CharSpan line) => ...`). Untyped +`LineReplace`); select them with an explicitly typed lambda parameter (`(ReadOnlySpan line) => ...`). Untyped lambdas continue to bind the existing `string`-based overloads. See [scrubbers.md](/docs/scrubbers.md) for full semantics. diff --git a/src/Verify/Serialization/Scrubbers/Scrubber.cs b/src/Verify/Serialization/Scrubbers/Scrubber.cs index b2595773cf..e4a642adf5 100644 --- a/src/Verify/Serialization/Scrubbers/Scrubber.cs +++ b/src/Verify/Serialization/Scrubbers/Scrubber.cs @@ -250,7 +250,7 @@ public static Scrubber RemoveLinesContaining(params string[] needles) => /// Remove any lines matching . /// No per line string is allocated for span predicates. /// Use an explicitly typed lambda parameter to select this overload, - /// e.g. RemoveLines((CharSpan line) => ...). + /// e.g. RemoveLines((ReadOnlySpan<char> line) => ...). /// [OverloadResolutionPriority(-1)] public static Scrubber RemoveLines(LineMatch shouldRemove) @@ -272,7 +272,7 @@ public static Scrubber RemoveLines(Func shouldRemove) /// Process each line via . /// No per line string is allocated for span based replacers. /// Use an explicitly typed lambda parameter to select this overload, - /// e.g. ReplaceLines((CharSpan line) => ...). + /// e.g. ReplaceLines((ReadOnlySpan<char> line) => ...). /// [OverloadResolutionPriority(-1)] public static Scrubber ReplaceLines(LineReplace replace) diff --git a/src/Verify/Serialization/Scrubbers/VerifierSettings_ExtensionMappedGlobalScrubbers.cs b/src/Verify/Serialization/Scrubbers/VerifierSettings_ExtensionMappedGlobalScrubbers.cs index fc1c061ffa..f16c5aa342 100644 --- a/src/Verify/Serialization/Scrubbers/VerifierSettings_ExtensionMappedGlobalScrubbers.cs +++ b/src/Verify/Serialization/Scrubbers/VerifierSettings_ExtensionMappedGlobalScrubbers.cs @@ -76,7 +76,7 @@ public static void ScrubLines(string extension, Func removeLine) = /// Remove any lines matching from the test results. /// No per line string is allocated for span predicates. /// Use an explicitly typed lambda parameter to select this overload, - /// e.g. ScrubLines(extension, (CharSpan line) => ...). + /// e.g. ScrubLines(extension, (ReadOnlySpan<char> line) => ...). /// [OverloadResolutionPriority(-1)] public static void ScrubLines(string extension, LineMatch removeLine) => @@ -105,7 +105,7 @@ public static void ScrubLinesWithReplace(string extension, Func /// Scrub lines with an optional replace. /// No per line string is allocated for span based replacers. /// Use an explicitly typed lambda parameter to select this overload, - /// e.g. ScrubLinesWithReplace(extension, (CharSpan line) => ...). + /// e.g. ScrubLinesWithReplace(extension, (ReadOnlySpan<char> line) => ...). /// [OverloadResolutionPriority(-1)] public static void ScrubLinesWithReplace(string extension, LineReplace replaceLine) => diff --git a/src/Verify/Serialization/Scrubbers/VerifierSettings_GlobalScrubbers.cs b/src/Verify/Serialization/Scrubbers/VerifierSettings_GlobalScrubbers.cs index 33abc4b1d0..07441bf67e 100644 --- a/src/Verify/Serialization/Scrubbers/VerifierSettings_GlobalScrubbers.cs +++ b/src/Verify/Serialization/Scrubbers/VerifierSettings_GlobalScrubbers.cs @@ -62,7 +62,7 @@ public static void ScrubLines(Func removeLine) => /// Remove any lines matching from the test results. /// No per line string is allocated for span predicates. /// Use an explicitly typed lambda parameter to select this overload, - /// e.g. ScrubLines((CharSpan line) => ...). + /// e.g. ScrubLines((ReadOnlySpan<char> line) => ...). /// [OverloadResolutionPriority(-1)] public static void ScrubLines(LineMatch removeLine) => @@ -142,7 +142,7 @@ public static void ScrubLinesWithReplace(Func replaceLine) => /// Scrub lines with an optional replace. /// No per line string is allocated for span based replacers. /// Use an explicitly typed lambda parameter to select this overload, - /// e.g. ScrubLinesWithReplace((CharSpan line) => ...). + /// e.g. ScrubLinesWithReplace((ReadOnlySpan<char> line) => ...). /// [OverloadResolutionPriority(-1)] public static void ScrubLinesWithReplace(LineReplace replaceLine) => diff --git a/src/Verify/Serialization/Scrubbers/VerifySettings_ExtensionMappedInstanceScrubbers.cs b/src/Verify/Serialization/Scrubbers/VerifySettings_ExtensionMappedInstanceScrubbers.cs index 5d79f0000c..2e4ae83660 100644 --- a/src/Verify/Serialization/Scrubbers/VerifySettings_ExtensionMappedInstanceScrubbers.cs +++ b/src/Verify/Serialization/Scrubbers/VerifySettings_ExtensionMappedInstanceScrubbers.cs @@ -92,7 +92,7 @@ public void ScrubLines(string extension, Func removeLine) => /// Remove any lines matching from the test results. /// No per line string is allocated for span predicates. /// Use an explicitly typed lambda parameter to select this overload, - /// e.g. ScrubLines(extension, (CharSpan line) => ...). + /// e.g. ScrubLines(extension, (ReadOnlySpan<char> line) => ...). /// [OverloadResolutionPriority(-1)] public void ScrubLines(string extension, LineMatch removeLine) => @@ -109,7 +109,7 @@ public void ScrubLinesWithReplace(string extension, Func replac /// Scrub lines with an optional replace. /// No per line string is allocated for span based replacers. /// Use an explicitly typed lambda parameter to select this overload, - /// e.g. ScrubLinesWithReplace(extension, (CharSpan line) => ...). + /// e.g. ScrubLinesWithReplace(extension, (ReadOnlySpan<char> line) => ...). /// [OverloadResolutionPriority(-1)] public void ScrubLinesWithReplace(string extension, LineReplace replaceLine) => diff --git a/src/Verify/Serialization/Scrubbers/VerifySettings_InstanceScrubbers.cs b/src/Verify/Serialization/Scrubbers/VerifySettings_InstanceScrubbers.cs index d78de599fb..68c4550de8 100644 --- a/src/Verify/Serialization/Scrubbers/VerifySettings_InstanceScrubbers.cs +++ b/src/Verify/Serialization/Scrubbers/VerifySettings_InstanceScrubbers.cs @@ -166,7 +166,7 @@ public void ScrubLines(Func removeLine) => /// Remove any lines matching from the test results. /// No per line string is allocated for span predicates. /// Use an explicitly typed lambda parameter to select this overload, - /// e.g. ScrubLines((CharSpan line) => ...). + /// e.g. ScrubLines((ReadOnlySpan<char> line) => ...). /// [OverloadResolutionPriority(-1)] public void ScrubLines(LineMatch removeLine) => @@ -183,7 +183,7 @@ public void ScrubLinesWithReplace(Func replaceLine) => /// Scrub lines with an optional replace. /// No per line string is allocated for span based replacers. /// Use an explicitly typed lambda parameter to select this overload, - /// e.g. ScrubLinesWithReplace((CharSpan line) => ...). + /// e.g. ScrubLinesWithReplace((ReadOnlySpan<char> line) => ...). /// [OverloadResolutionPriority(-1)] public void ScrubLinesWithReplace(LineReplace replaceLine) => From 961049ea5255c21c3567100920f914312a90c131 Mon Sep 17 00:00:00 2001 From: Simon Cropp Date: Sat, 18 Jul 2026 23:07:49 +1000 Subject: [PATCH 17/33] Update VerifyJsonWriter.cs --- src/Verify/Serialization/VerifyJsonWriter.cs | 36 +++++++++++++++++--- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/src/Verify/Serialization/VerifyJsonWriter.cs b/src/Verify/Serialization/VerifyJsonWriter.cs index e4bbe941e7..bb0587b3e1 100644 --- a/src/Verify/Serialization/VerifyJsonWriter.cs +++ b/src/Verify/Serialization/VerifyJsonWriter.cs @@ -57,8 +57,18 @@ public override void WriteValue(char value) base.WriteRawValue(value); } - public void WriteRawValueWithScrubbers(string value) => - WriteRawValueWithScrubbers(value.AsSpan()); + public void WriteRawValueWithScrubbers(string value) + { + if (value.Length == 0) + { + WriteRawValueIfNoStrict(value); + return; + } + + // The string overload returns the same instance when nothing changed, so + // the value the caller already holds is not duplicated + WriteRawValueIfNoStrict(ApplyScrubbers.ApplyForPropertyValue(value, settings, Counter)); + } public void WriteRawValueWithScrubbers(CharSpan value) { @@ -100,7 +110,21 @@ public override void WriteValue(string? value) return; } - WriteValue(value.AsSpan()); + if (value.Length == 0) + { + WriteRawValueIfNoStrict(value); + return; + } + + if (Counter.TryConvert(value.AsSpan(), out var result)) + { + WriteRawValueIfNoStrict(result); + return; + } + + // The string overload returns the same instance when nothing changed, so + // the value the caller already holds is not duplicated + WriteScrubbed(ApplyScrubbers.ApplyForPropertyValue(value, settings, Counter)); } public override void WriteValue(StringBuilder? value) => @@ -121,7 +145,11 @@ public override void WriteValue(CharSpan value) return; } - value = ApplyScrubbers.ApplyForPropertyValue(value, settings, Counter); + WriteScrubbed(ApplyScrubbers.ApplyForPropertyValue(value, settings, Counter)); + } + + void WriteScrubbed(CharSpan value) + { if (settings.StrictJson) { base.WriteValue(value); From 4cc5b01c59a77b4d38aad5f08435bfb41ab62971 Mon Sep 17 00:00:00 2001 From: Simon Cropp Date: Sat, 18 Jul 2026 23:22:44 +1000 Subject: [PATCH 18/33] . --- .../Serialization/Scrubbers/DateMatchers.cs | 24 +++----- .../Serialization/Scrubbers/GuidMatcher.cs | 8 +-- .../Serialization/Scrubbers/ScrubEngine.cs | 14 +++-- .../Serialization/Scrubbers/Scrubber.cs | 56 +++++++++++++------ 4 files changed, 58 insertions(+), 44 deletions(-) diff --git a/src/Verify/Serialization/Scrubbers/DateMatchers.cs b/src/Verify/Serialization/Scrubbers/DateMatchers.cs index 23a2826295..a153df87ee 100644 --- a/src/Verify/Serialization/Scrubbers/DateMatchers.cs +++ b/src/Verify/Serialization/Scrubbers/DateMatchers.cs @@ -130,34 +130,26 @@ static Scrubber Single(string format, Culture culture, ParseWindow parse) // prefilter analysis runs on what will actually render var digitPrefilter = HasDigitPrefilter(culture.DateTimeFormat.ExpandFormat(format)); - string? Match(CharSpan window, Counter counter, IReadOnlyDictionary context) - { - if (!counter.ScrubDateTimes) - { - return null; - } - - return parse(window, counter); - } + string? Match(CharSpan window, Counter counter, IReadOnlyDictionary context) => + parse(window, counter); if (digitPrefilter) { // The engine anchor jumps between digits, so no-match text is scanned // vectorized instead of per position - return Scrubber.AnchoredWindow( + return Scrubber.GatedWindow( Math.Max(1, min), max, Match, - requireWordBoundary: false, - anchor: WindowAnchor.Digit, - anchorChar: default, - anchorOffset: 0); + static counter => counter.ScrubDateTimes, + anchor: WindowAnchor.Digit); } - return Scrubber.Window( + return Scrubber.GatedWindow( Math.Max(1, min), max, - Match); + Match, + static counter => counter.ScrubDateTimes); } // True when the (expanded) format is guaranteed to render a digit first, diff --git a/src/Verify/Serialization/Scrubbers/GuidMatcher.cs b/src/Verify/Serialization/Scrubbers/GuidMatcher.cs index 33f6b189f0..a5f9b2c9a3 100644 --- a/src/Verify/Serialization/Scrubbers/GuidMatcher.cs +++ b/src/Verify/Serialization/Scrubbers/GuidMatcher.cs @@ -3,10 +3,11 @@ // format), so no-match text is scanned vectorized instead of per position. static class GuidMatcher { - public static readonly Scrubber Instance = Scrubber.AnchoredWindow( + public static readonly Scrubber Instance = Scrubber.GatedWindow( 36, 36, Match, + static counter => counter.ScrubGuids, requireWordBoundary: true, anchor: WindowAnchor.Char, anchorChar: '-', @@ -14,11 +15,6 @@ static class GuidMatcher static string? Match(CharSpan window, Counter counter, IReadOnlyDictionary context) { - if (!counter.ScrubGuids) - { - return null; - } - // Cheap prefilter: the "D" format has dashes at fixed offsets if (window[8] != '-' || window[13] != '-' || diff --git a/src/Verify/Serialization/Scrubbers/ScrubEngine.cs b/src/Verify/Serialization/Scrubbers/ScrubEngine.cs index 63d69b3bd4..1900b9e697 100644 --- a/src/Verify/Serialization/Scrubbers/ScrubEngine.cs +++ b/src/Verify/Serialization/Scrubbers/ScrubEngine.cs @@ -96,13 +96,19 @@ public static void RunToBuilder( changed = chunks != null; } - if (set.Inline.Length > 0) + foreach (var scrubber in set.Inline) { - chunks ??= [new(source, 0, source.Length, scannable: true)]; - foreach (var scrubber in set.Inline) + // A gated off built-in (inline dates or guids when the corresponding + // scrubbing is disabled) is skipped for the whole scrub rather than + // being probed at every candidate window + if (scrubber.Gate is { } gate && + !gate(counter)) { - changed |= ApplyInline(chunks, scrubber, counter, context); + continue; } + + chunks ??= [new(source, 0, source.Length, scannable: true)]; + changed |= ApplyInline(chunks, scrubber, counter, context); } // Path replacements are pinned last so user scrubbers always see raw paths diff --git a/src/Verify/Serialization/Scrubbers/Scrubber.cs b/src/Verify/Serialization/Scrubbers/Scrubber.cs index e4a642adf5..329f874124 100644 --- a/src/Verify/Serialization/Scrubbers/Scrubber.cs +++ b/src/Verify/Serialization/Scrubbers/Scrubber.cs @@ -38,6 +38,10 @@ public sealed class Scrubber internal LineReplace? LineReplacer { get; } internal Func? LineStringReplacer { get; } + // When set and it returns false, the engine skips this scrubber for the whole + // scrub rather than invoking it per candidate + internal Func? Gate { get; } + Scrubber( ScrubberKind kind, int minLength = 0, @@ -54,7 +58,8 @@ public sealed class Scrubber Func? lineStringReplacer = null, WindowAnchor anchor = WindowAnchor.None, char anchorChar = default, - int anchorOffset = 0) + int anchorOffset = 0, + Func? gate = null) { Kind = kind; MinLength = minLength; @@ -72,6 +77,7 @@ public sealed class Scrubber Anchor = anchor; AnchorChar = anchorChar; AnchorOffset = anchorOffset; + Gate = gate; } internal bool IsLineDrop => @@ -150,6 +156,18 @@ public static Scrubber Window( bool requireWordBoundary = false) { Ensure.NotNull(matcher); + ValidateWindowLengths(minLength, maxLength); + + return new( + ScrubberKind.Window, + minLength: minLength, + maxLength: maxLength, + requireWordBoundary: requireWordBoundary, + windowMatcher: matcher); + } + + static void ValidateWindowLengths(int minLength, int maxLength) + { if (minLength < 1) { throw new ArgumentOutOfRangeException(nameof(minLength), minLength, "minLength must be at least 1."); @@ -159,27 +177,27 @@ public static Scrubber Window( { throw new ArgumentOutOfRangeException(nameof(maxLength), maxLength, "maxLength must be greater than or equal to minLength."); } - - return new( - ScrubberKind.Window, - minLength: minLength, - maxLength: maxLength, - requireWordBoundary: requireWordBoundary, - windowMatcher: matcher); } - // A Window scrubber whose matches can only start where the anchor appears at - // anchorOffset from the window start. Used by the built-in guid and date - // scrubbers so no-match scans skip between candidate positions. - internal static Scrubber AnchoredWindow( + // A Window scrubber used by the built-in guid and date scrubbers. + // The gate is evaluated once per scrub, so a disabled built-in costs a single + // check instead of a full scan. + // When an anchor is supplied, matches can only start where it appears at + // anchorOffset from the window start, so no-match scans skip between candidate + // positions. + internal static Scrubber GatedWindow( int minLength, int maxLength, WindowMatch matcher, - bool requireWordBoundary, - WindowAnchor anchor, - char anchorChar, - int anchorOffset) => - new( + Func gate, + bool requireWordBoundary = false, + WindowAnchor anchor = WindowAnchor.None, + char anchorChar = default, + int anchorOffset = 0) + { + ValidateWindowLengths(minLength, maxLength); + + return new( ScrubberKind.Window, minLength: minLength, maxLength: maxLength, @@ -187,7 +205,9 @@ internal static Scrubber AnchoredWindow( windowMatcher: matcher, anchor: anchor, anchorChar: anchorChar, - anchorOffset: anchorOffset); + anchorOffset: anchorOffset, + gate: gate); + } /// /// Find matches using custom search logic. From 41d4cf5ed69951981e2c448f7dc954dbc1bbeb5e Mon Sep 17 00:00:00 2001 From: Simon Cropp Date: Sun, 19 Jul 2026 10:20:34 +1000 Subject: [PATCH 19/33] . --- docs/mdsource/scrubbers.source.md | 2 +- docs/scrubbers.md | 2 +- release-notes-32.0.0.md | 2 +- src/ApplyScrubbersTests/ComparisonTests.cs | 44 +++++++++ src/ApplyScrubbersTests/DeletionSeamTests.cs | 71 ++++++++++++++ src/ApplyScrubbersTests/DirectoryGateTests.cs | 30 ++++++ .../Serialization/Scrubbers/ScrubEngine.cs | 94 ++++++++++++++++++- .../Serialization/Scrubbers/Scrubber.cs | 30 +++++- 8 files changed, 267 insertions(+), 8 deletions(-) create mode 100644 src/ApplyScrubbersTests/ComparisonTests.cs create mode 100644 src/ApplyScrubbersTests/DeletionSeamTests.cs create mode 100644 src/ApplyScrubbersTests/DirectoryGateTests.cs diff --git a/docs/mdsource/scrubbers.source.md b/docs/mdsource/scrubbers.source.md index 9518e3fca9..53a8aeb16f 100644 --- a/docs/mdsource/scrubbers.source.md +++ b/docs/mdsource/scrubbers.source.md @@ -14,7 +14,7 @@ Scrubbing is performed by two mechanisms: A `Scrubber` is created via static factory methods and registered via `AddScrubber` at any [level](#Scrubber-levels): - * `Scrubber.Replace(find, replacement)`: replace every occurrence of a string. Supports a `StringComparison` and an optional word boundary requirement. A multi-pair overload replaces the longest matching find at any position. + * `Scrubber.Replace(find, replacement)`: replace every occurrence of a string. Supports an ordinal `StringComparison` (`Ordinal` or `OrdinalIgnoreCase`) and an optional word boundary requirement. A multi-pair overload replaces the longest matching find at any position. * `Scrubber.Window(minLength, maxLength, matcher)`: slide a window over the text; the matcher returns a replacement or null. Used by the inline guid and date scrubbers. * `Scrubber.Match(matcher, minLength, maxLength)`: custom search logic. The matcher locates the next match within a segment. * `Scrubber.RemoveLinesContaining(...)`, `Scrubber.RemoveLines(...)`, `Scrubber.ReplaceLines(...)`, `Scrubber.RemoveEmptyLines()`: line scoped scrubbers. diff --git a/docs/scrubbers.md b/docs/scrubbers.md index 0b3f019f75..8cf0b8cc15 100644 --- a/docs/scrubbers.md +++ b/docs/scrubbers.md @@ -21,7 +21,7 @@ Scrubbing is performed by two mechanisms: A `Scrubber` is created via static factory methods and registered via `AddScrubber` at any [level](#Scrubber-levels): - * `Scrubber.Replace(find, replacement)`: replace every occurrence of a string. Supports a `StringComparison` and an optional word boundary requirement. A multi-pair overload replaces the longest matching find at any position. + * `Scrubber.Replace(find, replacement)`: replace every occurrence of a string. Supports an ordinal `StringComparison` (`Ordinal` or `OrdinalIgnoreCase`) and an optional word boundary requirement. A multi-pair overload replaces the longest matching find at any position. * `Scrubber.Window(minLength, maxLength, matcher)`: slide a window over the text; the matcher returns a replacement or null. Used by the inline guid and date scrubbers. * `Scrubber.Match(matcher, minLength, maxLength)`: custom search logic. The matcher locates the next match within a segment. * `Scrubber.RemoveLinesContaining(...)`, `Scrubber.RemoveLines(...)`, `Scrubber.ReplaceLines(...)`, `Scrubber.RemoveEmptyLines()`: line scoped scrubbers. diff --git a/release-notes-32.0.0.md b/release-notes-32.0.0.md index 8f39c50425..3c57035941 100644 --- a/release-notes-32.0.0.md +++ b/release-notes-32.0.0.md @@ -102,7 +102,7 @@ Create a `Scrubber` via static factories and register it with `AddScrubber(Scrub (global, instance, extension mapped, fluent): ```csharp -// Fixed-string replacement (optional comparison / word boundary) +// Fixed-string replacement (optional ordinal comparison / word boundary) settings.AddScrubber(Scrubber.Replace("find", "replacement")); // Sliding window matcher (used internally by the guid and date scrubbers) diff --git a/src/ApplyScrubbersTests/ComparisonTests.cs b/src/ApplyScrubbersTests/ComparisonTests.cs new file mode 100644 index 0000000000..2b62dd3474 --- /dev/null +++ b/src/ApplyScrubbersTests/ComparisonTests.cs @@ -0,0 +1,44 @@ +// The engine splices out exactly Find.Length chars and skips text shorter than +// Find. Both only hold for ordinal comparisons. +public class ComparisonTests +{ + [Fact] + public void LinguisticComparisonRejectedForReplace() + { + var exception = Assert.Throws( + () => Scrubber.Replace("ab", "X", StringComparison.InvariantCulture)); + Assert.Contains("Ordinal", exception.Message); + + Assert.Throws( + () => Scrubber.Replace("ab", "X", StringComparison.CurrentCulture)); + Assert.Throws( + () => Scrubber.Replace("ab", "X", StringComparison.CurrentCultureIgnoreCase)); + Assert.Throws( + () => Scrubber.Replace(StringComparison.InvariantCultureIgnoreCase, false, ("ab", "X"))); + } + + [Fact] + public void OrdinalComparisonsAccepted() + { + Assert.Equal("X!", EngineRunner.Run("ab!", Scrubber.Replace("ab", "X"))); + Assert.Equal("X!", EngineRunner.Run("AB!", Scrubber.Replace("ab", "X", StringComparison.OrdinalIgnoreCase))); + } + + [Fact] + public void LinguisticNeedleDropsShorterLine() + { + // Soft hyphen is ignorable, so the 6 char line matches the 7 char needle. + // The line may not be skipped for being shorter than the needle. + var scrubber = Scrubber.RemoveLinesContaining(StringComparison.InvariantCultureIgnoreCase, "sec­ret"); + var result = EngineRunner.Run("keep\nsecret\nkeep2", scrubber); + Assert.Equal("keep\nkeep2", result); + } + + [Fact] + public void OrdinalNeedleDropsMatchingLine() + { + var scrubber = Scrubber.RemoveLinesContaining(StringComparison.Ordinal, "secret"); + var result = EngineRunner.Run("keep\nsecret\nkeep2", scrubber); + Assert.Equal("keep\nkeep2", result); + } +} diff --git a/src/ApplyScrubbersTests/DeletionSeamTests.cs b/src/ApplyScrubbersTests/DeletionSeamTests.cs new file mode 100644 index 0000000000..34a816798f --- /dev/null +++ b/src/ApplyScrubbersTests/DeletionSeamTests.cs @@ -0,0 +1,71 @@ +// An empty replacement quarantines nothing, so the text on either side of it is +// adjacent document text and must stay matchable by later scrubbers. +public class DeletionSeamTests +{ + static DeletionSeamTests() => + EngineRunner.UseFakeDirectories(); + + [Fact] + public void LaterScrubberMatchesAcrossDeletion() + { + var result = EngineRunner.Run( + "a--b", + Scrubber.Replace("--", ""), + Scrubber.Replace("ab", "X")); + Assert.Equal("X", result); + } + + [Fact] + public void LongerFindMatchesAcrossDeletion() + { + var result = EngineRunner.Run( + "daREDACT-ta", + Scrubber.Replace("REDACT-", ""), + Scrubber.Replace("data", "X")); + Assert.Equal("X", result); + } + + [Fact] + public void MultipleDeletionsAllJoined() + { + // Equal max lengths, so these run in registration order and the deletion + // happens first (inline scrubbers otherwise order by descending max length) + var result = EngineRunner.Run( + "aXXbXXcXXd", + Scrubber.Replace("XX", ""), + Scrubber.Replace("bc", "Y")); + Assert.Equal("aYd", result); + } + + [Fact] + public void DeletionAfterLinePhaseStillJoins() + { + var result = EngineRunner.Run( + "drop\na--b\nkeep", + Scrubber.RemoveLinesContaining(StringComparison.Ordinal, "drop"), + Scrubber.Replace("--", ""), + Scrubber.Replace("ab", "X")); + Assert.Equal("X\nkeep", result); + } + + [Fact] + public void DirectoryReplacementMatchesAcrossDeletion() + { + var result = EngineRunner.RunWithDirectoryReplacements( + "C:/Co[DEL]de/TheSolution/TheProject/file.txt", + Scrubber.Replace("[DEL]", "")); + Assert.Equal("{ProjectDirectory}file.txt", result); + } + + [Fact] + public void QuarantineSurvivesDeletion() + { + var result = EngineRunner.Run( + "a-b", + Scrubber.Replace("-", ""), + Scrubber.Replace("b", "a"), + Scrubber.Replace("aa", "BOOM")); + // The 'a' produced by the second scrubber is quarantined, so "aa" must not form + Assert.Equal("aa", result); + } +} diff --git a/src/ApplyScrubbersTests/DirectoryGateTests.cs b/src/ApplyScrubbersTests/DirectoryGateTests.cs new file mode 100644 index 0000000000..73424fcff0 --- /dev/null +++ b/src/ApplyScrubbersTests/DirectoryGateTests.cs @@ -0,0 +1,30 @@ +// Path replacement is pinned last, so it must run against the scrubbed document +// rather than being gated on the length of the pre-scrub source. +public class DirectoryGateTests +{ + static DirectoryGateTests() => + EngineRunner.UseFakeDirectories(); + + [Fact] + public void TransformExpandedTextIsPathScrubbed() + { + var result = EngineRunner.RunWithDirectoryReplacements( + "here", + Scrubber.ReplaceLines((string line) => line == "here" ? "C:/Code/TheSolution/TheProject/file.txt" : line)); + Assert.Equal("{ProjectDirectory}file.txt", result); + } + + [Fact] + public void ReplacementTextStaysQuarantined() + { + var result = EngineRunner.RunWithDirectoryReplacements( + "x", + Scrubber.Replace("x", "C:/Code/TheSolution/TheProject/f.txt")); + // Replacement text is quarantined, so it is deliberately not path scrubbed + Assert.Equal("C:/Code/TheSolution/TheProject/f.txt", result); + } + + [Fact] + public void ShortSourceWithNoScrubbersIsUnchanged() => + Assert.Equal("here", EngineRunner.RunWithDirectoryReplacements("here")); +} diff --git a/src/Verify/Serialization/Scrubbers/ScrubEngine.cs b/src/Verify/Serialization/Scrubbers/ScrubEngine.cs index 1900b9e697..9bae3569fa 100644 --- a/src/Verify/Serialization/Scrubbers/ScrubEngine.cs +++ b/src/Verify/Serialization/Scrubbers/ScrubEngine.cs @@ -115,11 +115,24 @@ public static void RunToBuilder( if (applyDirectoryReplacements) { var pairs = DirectoryReplacements.Items; - if (pairs.Count > 0 && - source.Length >= DirectoryReplacements.ShortestFindLength) + if (pairs.Count > 0) { - chunks ??= [new(source, 0, source.Length, scannable: true)]; - changed |= ApplyDirectoryReplacements(chunks, pairs); + if (chunks == null) + { + // Nothing has run, so the source is still the whole document + if (source.Length >= DirectoryReplacements.ShortestFindLength) + { + chunks = [new(source, 0, source.Length, scannable: true)]; + changed |= ApplyDirectoryReplacements(chunks, pairs); + } + } + else + { + // A scrubber may have grown the document past the shortest find, + // so the pre-scrub length cannot gate this. ApplyDirectoryReplacements + // skips any chunk that is too short on its own. + changed |= ApplyDirectoryReplacements(chunks, pairs); + } } } @@ -245,6 +258,7 @@ static bool ApplyInline( IReadOnlyDictionary context) { var changed = false; + var deleted = false; var effectiveMin = Math.Max(1, scrubber.MinLength); var chunkIndex = 0; while (chunkIndex < chunks.Count) @@ -264,12 +278,84 @@ static bool ApplyInline( } changed = true; + deleted |= replacement.Length == 0; chunkIndex = Splice(chunks, chunkIndex, matchStart, matchLength, replacement); } + if (deleted) + { + CoalesceScannable(chunks); + } + return changed; } + // An empty replacement quarantines nothing, so the document text on either + // side of it is now adjacent. Merging those chunks keeps later scrubbers, and + // the path replacement pass, able to match across the join. + // Only called after a pass that deleted, since the line phase legitimately + // leaves the whole document as adjacent scannable chunks and merging those + // would materialize it for no gain. + static void CoalesceScannable(List chunks) + { + var index = 0; + while (index < chunks.Count - 1) + { + if (!chunks[index].Scannable || + !chunks[index + 1].Scannable) + { + index++; + continue; + } + + var end = index + 1; + while (end + 1 < chunks.Count && + chunks[end + 1].Scannable) + { + end++; + } + + var merged = Merge(chunks, index, end); + chunks.RemoveRange(index, end - index + 1); + chunks.Insert(index, merged); + index++; + } + } + + static Chunk Merge(List chunks, int start, int end) + { + var first = chunks[start]; + var total = first.Length; + var contiguous = true; + for (var index = start + 1; index <= end; index++) + { + var chunk = chunks[index]; + var previous = chunks[index - 1]; + if (!ReferenceEquals(chunk.Text, previous.Text) || + previous.Start + previous.Length != chunk.Start) + { + contiguous = false; + } + + total += chunk.Length; + } + + // Slices of the same string that are already adjacent need no allocation + if (contiguous) + { + return new(first.Text, first.Start, total, scannable: true); + } + + var builder = new StringBuilder(total); + for (var index = start; index <= end; index++) + { + var chunk = chunks[index]; + builder.Append(chunk.Text, chunk.Start, chunk.Length); + } + + return new(builder.ToString(), 0, total, scannable: true); + } + // Splits the chunk at chunkIndex into [prefix][replacement][suffix]. // Returns the index of the suffix chunk (to continue scanning), or the index // after the inserted chunks when the match ended at the chunk end. diff --git a/src/Verify/Serialization/Scrubbers/Scrubber.cs b/src/Verify/Serialization/Scrubbers/Scrubber.cs index 329f874124..da4fcc9bd8 100644 --- a/src/Verify/Serialization/Scrubbers/Scrubber.cs +++ b/src/Verify/Serialization/Scrubbers/Scrubber.cs @@ -97,6 +97,7 @@ or ScrubberKind.Window /// /// Replace every occurrence of with . + /// must be or . /// public static Scrubber Replace( string find, @@ -106,6 +107,7 @@ public static Scrubber Replace( { ValidateFind(find); Ensure.NotNull(replacement); + ValidateOrdinal(comparison); return new( ScrubberKind.Replace, minLength: find.Length, @@ -118,6 +120,7 @@ public static Scrubber Replace( /// /// Replace every occurrence of each Find with its Replacement. /// At a given position the longest matching Find wins. + /// must be or . /// public static Scrubber Replace( StringComparison comparison, @@ -125,6 +128,7 @@ public static Scrubber Replace( params (string Find, string Replacement)[] pairs) { Ensure.NotNullOrEmpty(pairs); + ValidateOrdinal(comparison); var ordered = new (string Find, string Replacement)[pairs.Length]; for (var index = 0; index < pairs.Length; index++) { @@ -255,7 +259,9 @@ public static Scrubber RemoveLinesContaining(StringComparison comparison, params return new( ScrubberKind.LineDropNeedles, - minLength: minLength, + // A linguistic comparison can match a run shorter than the needle, so + // a line shorter than the needle cannot be skipped unread + minLength: IsOrdinal(comparison) ? minLength : 0, comparison: comparison, needles: copy); } @@ -329,6 +335,28 @@ internal static string NormalizeNewlines(string value) .Replace('\r', '\n'); } + // A replacement splices out exactly Find.Length chars, and the engine skips + // text shorter than Find. Both only hold for ordinal comparisons: a linguistic + // comparison can match a run of a different length than Find (for example an + // ignorable soft hyphen, or 'ss' matching 'ß'), which would splice the wrong + // range. + static void ValidateOrdinal(StringComparison comparison) + { + if (comparison is StringComparison.Ordinal or StringComparison.OrdinalIgnoreCase) + { + return; + } + + throw new ArgumentException( + $"Only Ordinal and OrdinalIgnoreCase are supported, but was {comparison}. A linguistic comparison can match a different number of characters than the find, so the match cannot be replaced reliably.", + nameof(comparison)); + } + + // True when a match is guaranteed to be the same length as the needle, so + // text shorter than the needle can be skipped without comparing + static bool IsOrdinal(StringComparison comparison) => + comparison is StringComparison.Ordinal or StringComparison.OrdinalIgnoreCase; + static void ValidateFind(string find) { Ensure.NotNullOrEmpty(find); From b0e1a201639e70b4029025c8b6fc0a67a4afb081 Mon Sep 17 00:00:00 2001 From: Simon Cropp Date: Sun, 19 Jul 2026 11:59:36 +1000 Subject: [PATCH 20/33] . --- src/ApplyScrubbersTests/CultureTests.cs | 101 +++++++++++++++ .../Serialization/Scrubbers/DateMatchers.cs | 117 ++++++++++++------ .../Serialization/Scrubbers/ScrubEngine.cs | 8 +- .../Serialization/Scrubbers/Scrubber.cs | 18 ++- 4 files changed, 201 insertions(+), 43 deletions(-) create mode 100644 src/ApplyScrubbersTests/CultureTests.cs diff --git a/src/ApplyScrubbersTests/CultureTests.cs b/src/ApplyScrubbersTests/CultureTests.cs new file mode 100644 index 0000000000..4c6b731982 --- /dev/null +++ b/src/ApplyScrubbersTests/CultureTests.cs @@ -0,0 +1,101 @@ +// Inline date scrubbers registered without an explicit culture must follow the +// culture in effect when the scrub runs, not the one in effect at registration. +public class CultureTests +{ + static readonly CultureInfo enUs = new("en-US"); + static readonly CultureInfo deDe = new("de-DE"); + + static string RunInCulture(CultureInfo culture, string input, params Scrubber[] scrubbers) + { + var original = CultureInfo.CurrentCulture; + try + { + CultureInfo.CurrentCulture = culture; + return EngineRunner.Run(input, scrubbers); + } + finally + { + CultureInfo.CurrentCulture = original; + } + } + + [Fact] + public void ScrubTimeCultureIsUsedWhenNoneSupplied() + { + // Registered under en-US, as a module initializer would be + var original = CultureInfo.CurrentCulture; + Scrubber[] scrubbers; + try + { + CultureInfo.CurrentCulture = enUs; + scrubbers = DateMatchers.DateTimes("dd MMMM yyyy", null); + } + finally + { + CultureInfo.CurrentCulture = original; + } + + var german = new DateTime(2024, 12, 5).ToString("dd MMMM yyyy", deDe); + var american = new DateTime(2024, 12, 5).ToString("dd MMMM yyyy", enUs); + + Assert.Equal("[DateTime_1]", RunInCulture(deDe, $"[{german}]", scrubbers)); + Assert.Equal("[DateTime_1]", RunInCulture(enUs, $"[{american}]", scrubbers)); + } + + [Fact] + public void ExplicitCultureIsNotAffectedByScrubTimeCulture() + { + var scrubbers = DateMatchers.DateTimes("dd MMMM yyyy", enUs); + + var american = new DateTime(2024, 12, 5).ToString("dd MMMM yyyy", enUs); + var german = new DateTime(2024, 12, 5).ToString("dd MMMM yyyy", deDe); + + // The explicit culture wins even while another culture is current + Assert.Equal("[DateTime_1]", RunInCulture(deDe, $"[{american}]", scrubbers)); + Assert.Equal($"[{german}]", RunInCulture(deDe, $"[{german}]", scrubbers)); + } + + [Fact] + public void CultureBoundsFollowScrubTimeCulture() + { + // Month name lengths differ between cultures, so the window bounds have to + // be rebuilt per culture and not just the parse + var original = CultureInfo.CurrentCulture; + Scrubber[] scrubbers; + try + { + CultureInfo.CurrentCulture = enUs; + scrubbers = DateMatchers.DateTimes("MMMM d yyyy", null); + } + finally + { + CultureInfo.CurrentCulture = original; + } + + var french = new CultureInfo("fr-FR"); + var rendered = new DateTime(2024, 2, 1).ToString("MMMM d yyyy", french); + Assert.Equal("[DateTime_1]", RunInCulture(french, $"[{rendered}]", scrubbers)); + } + + [Fact] + public void RepeatedScrubsReuseTheCachedCultureScrubber() + { + var original = CultureInfo.CurrentCulture; + Scrubber[] scrubbers; + try + { + CultureInfo.CurrentCulture = enUs; + scrubbers = DateMatchers.DateTimes("dd MMMM yyyy", null); + } + finally + { + CultureInfo.CurrentCulture = original; + } + + var german = new DateTime(2024, 12, 5).ToString("dd MMMM yyyy", deDe); + for (var index = 0; index < 3; index++) + { + Assert.Equal("[DateTime_1]", RunInCulture(deDe, $"[{german}]", scrubbers)); + } + } +} diff --git a/src/Verify/Serialization/Scrubbers/DateMatchers.cs b/src/Verify/Serialization/Scrubbers/DateMatchers.cs index a153df87ee..1212419b2c 100644 --- a/src/Verify/Serialization/Scrubbers/DateMatchers.cs +++ b/src/Verify/Serialization/Scrubbers/DateMatchers.cs @@ -3,7 +3,9 @@ // Formats ending in upper case fraction specifiers (.F to .FFFF) produce a second // scrubber for the trimmed format, since those fractions render as empty when zero; // the longer max of the untrimmed scrubber naturally orders it first. -// The culture (CurrentCulture when null) is resolved when the scrubber is created. +// An explicit culture is resolved when the scrubber is created. When none is given +// the culture in effect at scrub time is used, resolved (and cached) per culture on +// each scrub, since the parse, the window bounds, and the anchor all depend on it. static class DateMatchers { // A probe date within the supported range of every calendar @@ -26,19 +28,17 @@ public static Scrubber[] DateTimes( return BuildForFormats( format, + culture, resolvedCulture, - static (format, culture) => Single( - format, - culture, - (window, counter) => + static (format, culture) => (window, counter) => + { + if (DateTime.TryParseExact(window, format, culture, DateTimeStyles.None, out var date)) { - if (DateTime.TryParseExact(window, format, culture, DateTimeStyles.None, out var date)) - { - return counter.Convert(date); - } + return counter.Convert(date); + } - return null; - })); + return null; + }); } public static Scrubber[] DateTimeOffsets( @@ -57,19 +57,17 @@ public static Scrubber[] DateTimeOffsets( return BuildForFormats( format, + culture, resolvedCulture, - static (format, culture) => Single( - format, - culture, - (window, counter) => + static (format, culture) => (window, counter) => + { + if (DateTimeOffset.TryParseExact(window, format, culture, DateTimeStyles.None, out var date)) { - if (DateTimeOffset.TryParseExact(window, format, culture, DateTimeStyles.None, out var date)) - { - return counter.Convert(date); - } + return counter.Convert(date); + } - return null; - })); + return null; + }); } #if NET6_0_OR_GREATER @@ -90,40 +88,81 @@ public static Scrubber[] Dates( return BuildForFormats( format, + culture, resolvedCulture, - static (format, culture) => Single( - format, - culture, - (window, counter) => + static (format, culture) => (window, counter) => + { + if (Date.TryParseExact(window, format, culture, DateTimeStyles.None, out var date)) { - if (Date.TryParseExact(window, format, culture, DateTimeStyles.None, out var date)) - { - return counter.Convert(date); - } + return counter.Convert(date); + } - return null; - })); + return null; + }); } #endif delegate string? ParseWindow(CharSpan window, Counter counter); - static Scrubber[] BuildForFormats(string format, Culture culture, Func build) + // Builds the parse for one format and culture pair + delegate ParseWindow ParseFactory(string format, Culture culture); + + static Scrubber[] BuildForFormats( + string format, + Culture? culture, + Culture registrationCulture, + ParseFactory parseFactory) { if (TryGetFormatWithUpperMillisecondsTrimmed(format, out var trimmedFormat)) { return [ - build(format, culture), - build(trimmedFormat, culture) + ForCulture(format, culture, registrationCulture, parseFactory), + ForCulture(trimmedFormat, culture, registrationCulture, parseFactory) ]; } - return [build(format, culture)]; + return [ForCulture(format, culture, registrationCulture, parseFactory)]; + } + + static Scrubber ForCulture( + string format, + Culture? culture, + Culture registrationCulture, + ParseFactory parseFactory) + { + if (culture != null) + { + return Single(format, culture, parseFactory(format, culture)); + } + + // No culture was supplied, so each scrub uses the culture in effect at that + // point. Building a scrubber reads the format lengths and expands the + // pattern, so the result is cached per culture. + ConcurrentDictionary cache = new(); + + Scrubber ForCurrentCulture() + { + var current = Culture.CurrentCulture; + if (cache.TryGetValue(current, out var existing)) + { + return existing; + } + + return cache.GetOrAdd(current, Single(format, current, parseFactory(format, current))); + } + + // The registration culture instance supplies the bounds used for ordering + // and the length fast path; the resolver supplies the one actually run + return Single( + format, + registrationCulture, + parseFactory(format, registrationCulture), + ForCurrentCulture); } - static Scrubber Single(string format, Culture culture, ParseWindow parse) + static Scrubber Single(string format, Culture culture, ParseWindow parse, Func? resolver = null) { var (max, min) = DateFormatLengthCalculator.GetLength(format, culture); // Single char standard formats expand to the culture's pattern, so the @@ -142,14 +181,16 @@ static Scrubber Single(string format, Culture culture, ParseWindow parse) max, Match, static counter => counter.ScrubDateTimes, - anchor: WindowAnchor.Digit); + anchor: WindowAnchor.Digit, + resolver: resolver); } return Scrubber.GatedWindow( Math.Max(1, min), max, Match, - static counter => counter.ScrubDateTimes); + static counter => counter.ScrubDateTimes, + resolver: resolver); } // True when the (expanded) format is guaranteed to render a digit first, diff --git a/src/Verify/Serialization/Scrubbers/ScrubEngine.cs b/src/Verify/Serialization/Scrubbers/ScrubEngine.cs index 9bae3569fa..7a01724324 100644 --- a/src/Verify/Serialization/Scrubbers/ScrubEngine.cs +++ b/src/Verify/Serialization/Scrubbers/ScrubEngine.cs @@ -96,17 +96,21 @@ public static void RunToBuilder( changed = chunks != null; } - foreach (var scrubber in set.Inline) + foreach (var registered in set.Inline) { // A gated off built-in (inline dates or guids when the corresponding // scrubbing is disabled) is skipped for the whole scrub rather than // being probed at every candidate window - if (scrubber.Gate is { } gate && + if (registered.Gate is { } gate && !gate(counter)) { continue; } + // A scrubber registered without an explicit culture resolves here to + // the instance built for the culture in effect for this scrub + var scrubber = registered.Resolve(); + chunks ??= [new(source, 0, source.Length, scannable: true)]; changed |= ApplyInline(chunks, scrubber, counter, context); } diff --git a/src/Verify/Serialization/Scrubbers/Scrubber.cs b/src/Verify/Serialization/Scrubbers/Scrubber.cs index da4fcc9bd8..c316416e62 100644 --- a/src/Verify/Serialization/Scrubbers/Scrubber.cs +++ b/src/Verify/Serialization/Scrubbers/Scrubber.cs @@ -42,6 +42,14 @@ public sealed class Scrubber // scrub rather than invoking it per candidate internal Func? Gate { get; } + // When set, the instance actually run is resolved at scrub time. Used by the + // built-in date scrubbers, whose parse culture, window bounds, and anchor all + // depend on the culture in effect when the scrub runs. + internal Func? Resolver { get; } + + internal Scrubber Resolve() => + Resolver?.Invoke() ?? this; + Scrubber( ScrubberKind kind, int minLength = 0, @@ -59,7 +67,8 @@ public sealed class Scrubber WindowAnchor anchor = WindowAnchor.None, char anchorChar = default, int anchorOffset = 0, - Func? gate = null) + Func? gate = null, + Func? resolver = null) { Kind = kind; MinLength = minLength; @@ -78,6 +87,7 @@ public sealed class Scrubber AnchorChar = anchorChar; AnchorOffset = anchorOffset; Gate = gate; + Resolver = resolver; } internal bool IsLineDrop => @@ -197,7 +207,8 @@ internal static Scrubber GatedWindow( bool requireWordBoundary = false, WindowAnchor anchor = WindowAnchor.None, char anchorChar = default, - int anchorOffset = 0) + int anchorOffset = 0, + Func? resolver = null) { ValidateWindowLengths(minLength, maxLength); @@ -210,7 +221,8 @@ internal static Scrubber GatedWindow( anchor: anchor, anchorChar: anchorChar, anchorOffset: anchorOffset, - gate: gate); + gate: gate, + resolver: resolver); } /// From 089e6d794f32c12e67bf77675bd347aa1dc3bd6f Mon Sep 17 00:00:00 2001 From: Simon Cropp Date: Sun, 19 Jul 2026 12:07:24 +1000 Subject: [PATCH 21/33] . --- .../MultiLineTransformTests.cs | 85 +++++++++++++ .../Scrubbers/ScrubEngine_Lines.cs | 114 ++++++++++++++++-- 2 files changed, 191 insertions(+), 8 deletions(-) create mode 100644 src/ApplyScrubbersTests/MultiLineTransformTests.cs diff --git a/src/ApplyScrubbersTests/MultiLineTransformTests.cs b/src/ApplyScrubbersTests/MultiLineTransformTests.cs new file mode 100644 index 0000000000..ed608220c6 --- /dev/null +++ b/src/ApplyScrubbersTests/MultiLineTransformTests.cs @@ -0,0 +1,85 @@ +// A line transform is defined over a single line, so when one produces line +// breaks the remaining transforms must see each produced line on its own. +public class MultiLineTransformTests +{ + [Fact] + public void LaterTransformSeesProducedLines() + { + var result = EngineRunner.Run( + "a", + Scrubber.ReplaceLines((string line) => line == "a" ? "x\ny" : line), + Scrubber.ReplaceLines((string line) => line == "x" ? "z" : line)); + Assert.Equal("z\ny", result); + } + + [Fact] + public void LaterTransformSeesEveryProducedLine() + { + var result = EngineRunner.Run( + "a", + Scrubber.ReplaceLines((string line) => line == "a" ? "x\ny\nz" : line), + Scrubber.ReplaceLines((string line) => line.ToUpperInvariant())); + Assert.Equal("X\nY\nZ", result); + } + + [Fact] + public void LaterTransformCanRemoveAProducedLine() + { + var result = EngineRunner.Run( + "a", + Scrubber.ReplaceLines((string line) => line == "a" ? "keep\ndrop\nkeep2" : line), + Scrubber.ReplaceLines((string line) => line == "drop" ? null : line)); + Assert.Equal("keep\nkeep2", result); + } + + [Fact] + public void RemovingEveryProducedLineRemovesTheLine() + { + var result = EngineRunner.Run( + "a\nb", + Scrubber.ReplaceLines((string line) => line == "a" ? "x\ny" : line), + Scrubber.ReplaceLines((string line) => line is "x" or "y" ? null : line)); + Assert.Equal("b", result); + } + + [Fact] + public void ProducedLinesCanExpandAgain() + { + var result = EngineRunner.Run( + "a", + Scrubber.ReplaceLines((string line) => line == "a" ? "x\ny" : line), + Scrubber.ReplaceLines((string line) => line == "y" ? "1\n2" : line), + Scrubber.ReplaceLines((string line) => line == "2" ? "end" : line)); + Assert.Equal("x\n1\nend", result); + } + + [Fact] + public void SpanTransformSeesProducedLines() + { + var result = EngineRunner.Run( + "a", + Scrubber.ReplaceLines((CharSpan line) => line.SequenceEqual("a".AsSpan()) ? LineResult.Replace("x\ny") : LineResult.Keep), + Scrubber.ReplaceLines((CharSpan line) => line.SequenceEqual("x".AsSpan()) ? LineResult.Replace("z") : LineResult.Keep)); + Assert.Equal("z\ny", result); + } + + [Fact] + public void TrailingTransformLeavesTextExactlyAsProduced() + { + // Nothing follows the expanding transform, so the text is untouched + var result = EngineRunner.Run( + "a", + Scrubber.ReplaceLines((string line) => line == "a" ? "x\ny\n" : line)); + Assert.Equal("x\ny\n", result); + } + + [Fact] + public void SurroundingLinesArePreserved() + { + var result = EngineRunner.Run( + "before\na\nafter", + Scrubber.ReplaceLines((string line) => line == "a" ? "x\ny" : line), + Scrubber.ReplaceLines((string line) => line == "x" ? "z" : line)); + Assert.Equal("before\nz\ny\nafter", result); + } +} diff --git a/src/Verify/Serialization/Scrubbers/ScrubEngine_Lines.cs b/src/Verify/Serialization/Scrubbers/ScrubEngine_Lines.cs index 1789a971ad..76b229b19f 100644 --- a/src/Verify/Serialization/Scrubbers/ScrubEngine_Lines.cs +++ b/src/Verify/Serialization/Scrubbers/ScrubEngine_Lines.cs @@ -1,6 +1,8 @@ // The line phase: walks the (already newline-normalized) source once, applying line drops first // (needle, whitespace, and predicate based) then line transforms in registration order. Drops always // evaluate the raw line; transform output becomes fresh scannable source for the inline phase. +// A transform that returns line breaks produces several lines, and the transforms after it see each +// of those on its own, since a line transform is defined over a single line. // Join semantics replicate the legacy StringReader based line scrubbers: lines are joined with \n, // the trailing newline is preserved only when the original ended with one, and RemoveEmptyLines // additionally trims the trailing newline. @@ -192,8 +194,10 @@ static bool IsDropped(CharSpan lineSpan, ref string? lineString, Scrubber[] line { // The most recent replacement text; null while the line is unchanged string? current = null; - foreach (var transform in lineTransforms) + for (var index = 0; index < lineTransforms.Length; index++) { + var transform = lineTransforms[index]; + string output; switch (transform.Kind) { case ScrubberKind.LineTransformSpan: @@ -204,30 +208,124 @@ static bool IsDropped(CharSpan lineSpan, ref string? lineString, Scrubber[] line return (true, null); } - if (result.Kind == LineResult.ReplaceKind) + if (result.Kind != LineResult.ReplaceKind) { - current = result.Text!; + continue; } - continue; + output = result.Text!; + break; } case ScrubberKind.LineTransformString: { var input = current ?? (lineString ??= lineSpan.ToString()); - var output = transform.LineStringReplacer!(input); - if (output == null) + var replaced = transform.LineStringReplacer!(input); + if (replaced == null) { return (true, null); } - current = Scrubber.NormalizeNewlines(output); - continue; + output = Scrubber.NormalizeNewlines(replaced); + break; } default: throw new($"Unexpected line transform kind: {transform.Kind}"); } + + current = output; + + // A replacement holding line breaks is several lines, and a line + // transform is defined over one line, so the remaining transforms see + // each produced line on its own. The pre engine pipeline did this by + // re-reading the document between passes. + if (output.Contains('\n')) + { + return ApplyToProducedLines(output, lineTransforms, index + 1); + } } return (false, current); } + + // The remaining transforms over each line of a multi line replacement. + // Splitting and joining on '\n' are exact inverses, so transforms that change + // nothing leave the text as it was. + static (bool removed, string? current) ApplyToProducedLines(string text, Scrubber[] lineTransforms, int startIndex) + { + if (startIndex == lineTransforms.Length) + { + return (false, text); + } + + var lines = new List(text.Split('\n')); + for (var index = startIndex; index < lineTransforms.Length; index++) + { + var transform = lineTransforms[index]; + var next = new List(lines.Count); + foreach (var line in lines) + { + if (!TryTransformLine(transform, line, out var replacement)) + { + continue; + } + + if (replacement == null) + { + next.Add(line); + continue; + } + + if (replacement.Contains('\n')) + { + next.AddRange(replacement.Split('\n')); + continue; + } + + next.Add(replacement); + } + + lines = next; + } + + if (lines.Count == 0) + { + return (true, null); + } + + return (false, string.Join("\n", lines)); + } + + // False when the line is removed. replacement is null when it is unchanged. + static bool TryTransformLine(Scrubber transform, string line, out string? replacement) + { + switch (transform.Kind) + { + case ScrubberKind.LineTransformSpan: + { + var result = transform.LineReplacer!(line.AsSpan()); + if (result.Kind == LineResult.RemoveKind) + { + replacement = null; + return false; + } + + replacement = result.Kind == LineResult.ReplaceKind ? result.Text! : null; + return true; + } + case ScrubberKind.LineTransformString: + { + var output = transform.LineStringReplacer!(line); + if (output == null) + { + replacement = null; + return false; + } + + replacement = Scrubber.NormalizeNewlines(output); + return true; + } + default: + throw new($"Unexpected line transform kind: {transform.Kind}"); + } + } } From e4d457be2ae9d3c641aca958d5c77e6de9e94633 Mon Sep 17 00:00:00 2001 From: Simon Cropp Date: Sun, 19 Jul 2026 17:41:23 +1000 Subject: [PATCH 22/33] . --- src/ApplyScrubbersTests/DigitAnchorTests.cs | 58 +++++++++++++++++ .../Serialization/Scrubbers/DateMatchers.cs | 63 ++++++++++++++++--- .../Serialization/Scrubbers/ScrubEngine.cs | 17 +++-- src/Verify/SettingsTask_Scrubbing.cs | 9 +++ .../SettingsTask_Scrubbing_Obsoletes.cs | 9 --- 5 files changed, 130 insertions(+), 26 deletions(-) create mode 100644 src/ApplyScrubbersTests/DigitAnchorTests.cs diff --git a/src/ApplyScrubbersTests/DigitAnchorTests.cs b/src/ApplyScrubbersTests/DigitAnchorTests.cs new file mode 100644 index 0000000000..ab3b6dab58 --- /dev/null +++ b/src/ApplyScrubbersTests/DigitAnchorTests.cs @@ -0,0 +1,58 @@ +// The digit anchor lets the engine jump between digit positions instead of +// probing every one. It may only be used when the format really does render an +// ASCII digit first, otherwise candidate windows are never probed. +public class DigitAnchorTests +{ + [Fact] + public void LeadingOptionalFractionIsScrubbed() + { + // Upper case F renders nothing when the fraction is zero, so the value + // starts with the literal space and a digit anchor would never probe it + var rendered = new DateTime(2020, 1, 1, 0, 30, 0).ToString("FF mm", CultureInfo.InvariantCulture); + Assert.Equal(" 30", rendered); + + using var counter = Counter.Start(); + var scrubbers = DateMatchers.DateTimes("FF mm", CultureInfo.InvariantCulture); + Assert.Equal("[DateTime_1]", EngineRunner.Run($"[{rendered}]", counter, scrubbers)); + } + + [Fact] + public void NonZeroFractionIsScrubbed() + { + var date = new DateTime(2020, 1, 1, 0, 30, 0).AddMilliseconds(250); + var rendered = date.ToString("FF mm", CultureInfo.InvariantCulture); + Assert.Equal("25 30", rendered); + + using var counter = Counter.Start(); + var scrubbers = DateMatchers.DateTimes("FF mm", CultureInfo.InvariantCulture); + Assert.Equal("[DateTime_1]", EngineRunner.Run($"[{rendered}]", counter, scrubbers)); + } + + [Fact] + public void DigitLeadingFormatIsScrubbed() + { + using var counter = Counter.Start(); + var scrubbers = DateMatchers.DateTimes("yyyy-MM-dd", CultureInfo.InvariantCulture); + Assert.Equal("[DateTime_1]", EngineRunner.Run("[2024-01-02]", counter, scrubbers)); + } + + [Fact] + public void NameLeadingFormatIsScrubbed() + { + using var counter = Counter.Start(); + var scrubbers = DateMatchers.DateTimes("MMMM d yyyy", CultureInfo.InvariantCulture); + Assert.Equal("[DateTime_1]", EngineRunner.Run("[January 2 2024]", counter, scrubbers)); + } + + [Fact] + public void LowerCaseFractionIsScrubbed() + { + // Lower case f always renders digits, so the anchor stays available + var rendered = new DateTime(2020, 1, 1, 0, 30, 0).ToString("ff mm", CultureInfo.InvariantCulture); + Assert.Equal("00 30", rendered); + + using var counter = Counter.Start(); + var scrubbers = DateMatchers.DateTimes("ff mm", CultureInfo.InvariantCulture); + Assert.Equal("[DateTime_1]", EngineRunner.Run($"[{rendered}]", counter, scrubbers)); + } +} diff --git a/src/Verify/Serialization/Scrubbers/DateMatchers.cs b/src/Verify/Serialization/Scrubbers/DateMatchers.cs index 1212419b2c..6e694d86d2 100644 --- a/src/Verify/Serialization/Scrubbers/DateMatchers.cs +++ b/src/Verify/Serialization/Scrubbers/DateMatchers.cs @@ -165,9 +165,7 @@ Scrubber ForCurrentCulture() static Scrubber Single(string format, Culture culture, ParseWindow parse, Func? resolver = null) { var (max, min) = DateFormatLengthCalculator.GetLength(format, culture); - // Single char standard formats expand to the culture's pattern, so the - // prefilter analysis runs on what will actually render - var digitPrefilter = HasDigitPrefilter(culture.DateTimeFormat.ExpandFormat(format)); + var digitPrefilter = HasDigitPrefilter(format, culture); string? Match(CharSpan window, Counter counter, IReadOnlyDictionary context) => parse(window, counter); @@ -193,10 +191,59 @@ static Scrubber Single(string format, Culture culture, ParseWindow parse, Func + value is >= '0' and <= '9'; + + // True when the (expanded) format starts with a token that renders a number. + // Formats starting with a name, era, or literal token are not prefiltered. + // Upper case F is excluded since it renders nothing when the fraction is zero. + static bool StartsWithNumericToken(string format) { if (format.Length < 2) { @@ -206,7 +253,7 @@ static bool HasDigitPrefilter(string format) var first = format[0]; switch (first) { - case 'y' or 'H' or 'h' or 'm' or 's' or 'f' or 'F': + case 'y' or 'H' or 'h' or 'm' or 's' or 'f': return true; case 'd' or 'M': // 1 or 2 repeats render digits; 3+ render names diff --git a/src/Verify/Serialization/Scrubbers/ScrubEngine.cs b/src/Verify/Serialization/Scrubbers/ScrubEngine.cs index 7a01724324..367456e1bb 100644 --- a/src/Verify/Serialization/Scrubbers/ScrubEngine.cs +++ b/src/Verify/Serialization/Scrubbers/ScrubEngine.cs @@ -548,6 +548,10 @@ static bool TryFindWindowMatch( return false; } +#if !NET8_0_OR_GREATER + static string asciiDigits = "0123456789"; +#endif + // The next position at or after the given one where the scrubber's anchor // appears at its offset from the window start. Returns past regionEnd when no // candidate remains. @@ -567,18 +571,13 @@ static int NextAnchoredPosition(CharSpan span, int position, int regionEnd, Scru } else { + // ASCII only on every target: char.IsDigit would also accept other + // Unicode digits, so the same input would anchor differently per + // target framework while sharing one verified file #if NET8_0_OR_GREATER found = region.IndexOfAnyInRange('0', '9'); #else - found = -1; - for (var index = 0; index < region.Length; index++) - { - if (char.IsDigit(region[index])) - { - found = index; - break; - } - } + found = region.IndexOfAny(asciiDigits.AsSpan()); #endif } diff --git a/src/Verify/SettingsTask_Scrubbing.cs b/src/Verify/SettingsTask_Scrubbing.cs index 7fc28525fa..2fc9ce4d8a 100644 --- a/src/Verify/SettingsTask_Scrubbing.cs +++ b/src/Verify/SettingsTask_Scrubbing.cs @@ -197,6 +197,15 @@ public SettingsTask ScrubLines(string extension, Func removeLine) return this; } + /// + [OverloadResolutionPriority(-1)] + [Pure] + public SettingsTask ScrubLines(string extension, LineMatch removeLine) + { + CurrentSettings.ScrubLines(extension, removeLine); + return this; + } + /// [Pure] public SettingsTask ScrubLinesWithReplace(Func replaceLine) diff --git a/src/Verify/SettingsTask_Scrubbing_Obsoletes.cs b/src/Verify/SettingsTask_Scrubbing_Obsoletes.cs index c468cb0194..4692d5423c 100644 --- a/src/Verify/SettingsTask_Scrubbing_Obsoletes.cs +++ b/src/Verify/SettingsTask_Scrubbing_Obsoletes.cs @@ -91,15 +91,6 @@ public SettingsTask ScrubLinesContaining(string extension, StringComparison comp public SettingsTask ScrubLines(Func removeLine, ScrubberLocation location) => ScrubLines(removeLine); - /// - [OverloadResolutionPriority(-1)] - [Pure] - public SettingsTask ScrubLines(string extension, LineMatch removeLine) - { - CurrentSettings.ScrubLines(extension, removeLine); - return this; - } - /// [Obsolete(locationObsolete)] [Pure] From 2f4f4d4acb6549da886fecbaf09195f7427a1b43 Mon Sep 17 00:00:00 2001 From: Simon Cropp Date: Sun, 19 Jul 2026 18:37:10 +1000 Subject: [PATCH 23/33] . --- src/Benchmarks/TempPathScrubberBenchmarks.cs | 153 +++++++++++++++++++ src/Verify/TempDirectory.cs | 37 +---- src/Verify/TempFile.cs | 37 +---- src/Verify/TempPathScrubber.cs | 51 +++++++ 4 files changed, 206 insertions(+), 72 deletions(-) create mode 100644 src/Benchmarks/TempPathScrubberBenchmarks.cs create mode 100644 src/Verify/TempPathScrubber.cs diff --git a/src/Benchmarks/TempPathScrubberBenchmarks.cs b/src/Benchmarks/TempPathScrubberBenchmarks.cs new file mode 100644 index 0000000000..039fba7419 --- /dev/null +++ b/src/Benchmarks/TempPathScrubberBenchmarks.cs @@ -0,0 +1,153 @@ +// TempDirectory and TempFile each held a verbatim copy of the tracked-path matcher. +// Duplicated_* rows run that copy, built inline so it closes over statics only. +// Shared_* rows run the extracted TempPathScrubber.Build, whose matcher closes over +// its parameters instead. The question is whether that extra closure indirection +// costs anything on the scrub path. +// Document sizes follow the *.verified.* scan used by the other scrubber benchmarks: +// p50=260 chars, p90=2.9KB, p99=31KB. NoPaths is the dominant real case, since the +// async local is null unless a test actually created a temp path. +[MemoryDiagnoser] +[SimpleJob(iterationCount: 10, warmupCount: 3)] +public class TempPathScrubberBenchmarks +{ + static AsyncLocal?> asyncPaths = new(); + static readonly object pathsLock = new(); + + // A realistic root, matching [System.Temp]\VerifyTempDirectory + static string rootDirectory = Path.Combine(Path.GetTempPath(), "VerifyTempDirectory"); + static string trackedPath = Path.Combine(rootDirectory, "abcd1234.xyz"); + + static Dictionary emptyContext = []; + + string small = null!; + string medium = null!; + string large = null!; + string largeWithPaths = null!; + + EngineScrubberSet duplicatedSet = null!; + EngineScrubberSet sharedSet = null!; + + [GlobalSetup] + public void Setup() + { + small = Build(260, false); + medium = Build(2_900, false); + large = Build(31_000, false); + largeWithPaths = Build(31_000, true); + + duplicatedSet = EngineScrubberSet.ForScrubbers([BuildDuplicated()]); + sharedSet = EngineScrubberSet.ForScrubbers([TempPathScrubber.Build(asyncPaths, pathsLock, "{TempDirectory}", rootDirectory.Length)]); + } + + // Lines of ~40 chars; when withPaths, every 12th line embeds the tracked path + static string Build(int targetChars, bool withPaths) + { + var builder = new StringBuilder(); + var line = 0; + while (builder.Length < targetChars) + { + if (withPaths && + line % 12 == 0) + { + builder.Append(" \"path\": \""); + builder.Append(trackedPath); + builder.Append("\","); + } + else + { + builder.Append(" \"name\": \"some value here\","); + } + + builder.Append('\n'); + line++; + } + + return builder.ToString(); + } + + // The matcher as it was duplicated in TempDirectory.Init and TempFile.Init + static Scrubber BuildDuplicated() => + Scrubber.Match( + (ReadOnlySpan segment, Counter _, IReadOnlyDictionary _, out int index, out int length, out string? replacement) => + { + index = -1; + length = 0; + replacement = "{TempDirectory}"; + var pathsValue = asyncPaths.Value; + if (pathsValue == null) + { + return false; + } + + lock (pathsLock) + { + foreach (var path in pathsValue) + { + var found = segment.IndexOf(path.AsSpan(), StringComparison.Ordinal); + if (found < 0) + { + continue; + } + + if (index < 0 || + found < index) + { + index = found; + length = path.Length; + } + } + } + + return index >= 0; + }, + minLength: rootDirectory.Length); + + static string Run(string input, EngineScrubberSet set) + { + using var counter = Counter.Start(); + return ScrubEngine.Run(input, set, counter, emptyContext, applyDirectoryReplacements: false); + } + + static string RunWithPaths(string input, EngineScrubberSet set) + { + asyncPaths.Value = [trackedPath]; + try + { + return Run(input, set); + } + finally + { + asyncPaths.Value = null; + } + } + + [Benchmark] + public string Duplicated_NoPaths_Small() => Run(small, duplicatedSet); + + [Benchmark] + public string Shared_NoPaths_Small() => Run(small, sharedSet); + + [Benchmark] + public string Duplicated_NoPaths_Medium() => Run(medium, duplicatedSet); + + [Benchmark] + public string Shared_NoPaths_Medium() => Run(medium, sharedSet); + + [Benchmark] + public string Duplicated_NoPaths_Large() => Run(large, duplicatedSet); + + [Benchmark] + public string Shared_NoPaths_Large() => Run(large, sharedSet); + + [Benchmark] + public string Duplicated_PathsNoMatch_Large() => RunWithPaths(large, duplicatedSet); + + [Benchmark] + public string Shared_PathsNoMatch_Large() => RunWithPaths(large, sharedSet); + + [Benchmark] + public string Duplicated_PathsMatch_Large() => RunWithPaths(largeWithPaths, duplicatedSet); + + [Benchmark] + public string Shared_PathsMatch_Large() => RunWithPaths(largeWithPaths, sharedSet); +} diff --git a/src/Verify/TempDirectory.cs b/src/Verify/TempDirectory.cs index edc754de9c..510bef0d19 100644 --- a/src/Verify/TempDirectory.cs +++ b/src/Verify/TempDirectory.cs @@ -63,43 +63,8 @@ public static void Init() }, after: () => asyncPaths.Value = null); - // All temp directory paths start with RootDirectory, so segments shorter - // than that can be skipped without invoking the matcher VerifierSettings.AddScrubber( - Scrubber.Match( - (CharSpan segment, Counter _, IReadOnlyDictionary _, out int index, out int length, out string? replacement) => - { - index = -1; - length = 0; - replacement = "{TempDirectory}"; - var pathsValue = asyncPaths.Value; - if (pathsValue == null) - { - return false; - } - - lock (pathsLock) - { - foreach (var path in pathsValue) - { - var found = segment.IndexOf(path.AsSpan(), StringComparison.Ordinal); - if (found < 0) - { - continue; - } - - if (index < 0 || - found < index) - { - index = found; - length = path.Length; - } - } - } - - return index >= 0; - }, - minLength: RootDirectory.Length)); + TempPathScrubber.Build(asyncPaths, pathsLock, "{TempDirectory}", RootDirectory.Length)); Cleanup(); } diff --git a/src/Verify/TempFile.cs b/src/Verify/TempFile.cs index f5972fb649..54f4043b8b 100644 --- a/src/Verify/TempFile.cs +++ b/src/Verify/TempFile.cs @@ -65,43 +65,8 @@ public static void Init() }, after: () => asyncPaths.Value = null); - // All temp file paths start with RootDirectory, so segments shorter than - // that can be skipped without invoking the matcher VerifierSettings.AddScrubber( - Scrubber.Match( - (CharSpan segment, Counter _, IReadOnlyDictionary _, out int index, out int length, out string? replacement) => - { - index = -1; - length = 0; - replacement = "{TempFile}"; - var pathsValue = asyncPaths.Value; - if (pathsValue == null) - { - return false; - } - - lock (pathsLock) - { - foreach (var path in pathsValue) - { - var found = segment.IndexOf(path.AsSpan(), StringComparison.Ordinal); - if (found < 0) - { - continue; - } - - if (index < 0 || - found < index) - { - index = found; - length = path.Length; - } - } - } - - return index >= 0; - }, - minLength: RootDirectory.Length)); + TempPathScrubber.Build(asyncPaths, pathsLock, "{TempFile}", RootDirectory.Length)); Cleanup(); } diff --git a/src/Verify/TempPathScrubber.cs b/src/Verify/TempPathScrubber.cs new file mode 100644 index 0000000000..1de6316f10 --- /dev/null +++ b/src/Verify/TempPathScrubber.cs @@ -0,0 +1,51 @@ +namespace VerifyTests; + +// The scrubber shared by TempDirectory and TempFile. Each tracks the paths it has +// created for the current async context, so the paths are only known at scrub time +// and cannot be expressed as fixed Replace pairs. +static class TempPathScrubber +{ + // Replaces any tracked path with token. The leftmost match in the segment wins, + // matching the engine's leftmost first scan. + // All tracked paths start with the root directory, so segments shorter than it + // are skipped without invoking the matcher. + public static Scrubber Build( + AsyncLocal?> paths, + object pathsLock, + string token, + int rootDirectoryLength) => + Scrubber.Match( + (CharSpan segment, Counter _, IReadOnlyDictionary _, out int index, out int length, out string? replacement) => + { + index = -1; + length = 0; + replacement = token; + var pathsValue = paths.Value; + if (pathsValue == null) + { + return false; + } + + lock (pathsLock) + { + foreach (var path in pathsValue) + { + var found = segment.IndexOf(path.AsSpan(), StringComparison.Ordinal); + if (found < 0) + { + continue; + } + + if (index < 0 || + found < index) + { + index = found; + length = path.Length; + } + } + } + + return index >= 0; + }, + minLength: rootDirectoryLength); +} From d1dcb38fb802c4476bab9858627e7395ac3a21a5 Mon Sep 17 00:00:00 2001 From: Simon Cropp Date: Sun, 19 Jul 2026 18:59:04 +1000 Subject: [PATCH 24/33] . --- src/Benchmarks/SpliceBenchmarks.cs | 82 ++++++ .../Serialization/Scrubbers/ScrubEngine.cs | 277 +++++++++++------- 2 files changed, 247 insertions(+), 112 deletions(-) create mode 100644 src/Benchmarks/SpliceBenchmarks.cs diff --git a/src/Benchmarks/SpliceBenchmarks.cs b/src/Benchmarks/SpliceBenchmarks.cs new file mode 100644 index 0000000000..a45195bd47 --- /dev/null +++ b/src/Benchmarks/SpliceBenchmarks.cs @@ -0,0 +1,82 @@ +// Splice removes and re-inserts in the shared chunk list, so every match shifts the +// elements after it. That is free while the list is short or matches land at the +// tail, but a second scrubber running over a list a first scrubber already +// fragmented splices near the front of a long list, giving O(matches x chunks). +// Fragmented_* rows are that shape: scrubber A (longer find, so it runs first) +// splits the document into ~2N chunks, then scrubber B matches N times across them. +// Single_* rows are the control: B alone over an unfragmented document, where +// splices land at the tail and cost nothing. +// Sizes step by 4x so a quadratic term shows as a ~16x step. +[MemoryDiagnoser] +[SimpleJob(iterationCount: 10, warmupCount: 3)] +public class SpliceBenchmarks +{ + const string longToken = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; // 40 chars + const string shortToken = "BBBBBBBBBB"; // 10 chars + + static Dictionary emptyContext = []; + + string small = null!; + string medium = null!; + string large = null!; + + EngineScrubberSet fragmentingSet = null!; + EngineScrubberSet singleSet = null!; + + [GlobalSetup] + public void Setup() + { + small = Build(200); + medium = Build(800); + large = Build(3_200); + + // A has the longer find so the engine orders it first, fragmenting the list + // before B runs over it + fragmentingSet = EngineScrubberSet.ForScrubbers( + [ + Scrubber.Replace(longToken, "{A}"), + Scrubber.Replace(shortToken, "{B}") + ]); + singleSet = EngineScrubberSet.ForScrubbers([Scrubber.Replace(shortToken, "{B}")]); + } + + // Each line holds one long token and one short token + static string Build(int lines) + { + var builder = new StringBuilder(); + for (var line = 0; line < lines; line++) + { + builder.Append(" \"a\": \""); + builder.Append(longToken); + builder.Append("\", \"b\": \""); + builder.Append(shortToken); + builder.Append("\",\n"); + } + + return builder.ToString(); + } + + static string Run(string input, EngineScrubberSet set) + { + using var counter = Counter.Start(); + return ScrubEngine.Run(input, set, counter, emptyContext, applyDirectoryReplacements: false); + } + + [Benchmark] + public string Fragmented_200() => Run(small, fragmentingSet); + + [Benchmark] + public string Fragmented_800() => Run(medium, fragmentingSet); + + [Benchmark] + public string Fragmented_3200() => Run(large, fragmentingSet); + + [Benchmark] + public string Single_200() => Run(small, singleSet); + + [Benchmark] + public string Single_800() => Run(medium, singleSet); + + [Benchmark] + public string Single_3200() => Run(large, singleSet); +} diff --git a/src/Verify/Serialization/Scrubbers/ScrubEngine.cs b/src/Verify/Serialization/Scrubbers/ScrubEngine.cs index 367456e1bb..cb01ed25e7 100644 --- a/src/Verify/Serialization/Scrubbers/ScrubEngine.cs +++ b/src/Verify/Serialization/Scrubbers/ScrubEngine.cs @@ -112,7 +112,11 @@ public static void RunToBuilder( var scrubber = registered.Resolve(); chunks ??= [new(source, 0, source.Length, scannable: true)]; - changed |= ApplyInline(chunks, scrubber, counter, context); + if (ApplyInline(chunks, scrubber, counter, context) is { } applied) + { + chunks = applied; + changed = true; + } } // Path replacements are pinned last so user scrubbers always see raw paths @@ -127,7 +131,11 @@ public static void RunToBuilder( if (source.Length >= DirectoryReplacements.ShortestFindLength) { chunks = [new(source, 0, source.Length, scannable: true)]; - changed |= ApplyDirectoryReplacements(chunks, pairs); + if (ApplyDirectoryReplacements(chunks, pairs) is { } replaced) + { + chunks = replaced; + changed = true; + } } } else @@ -135,7 +143,11 @@ public static void RunToBuilder( // A scrubber may have grown the document past the shortest find, // so the pre-scrub length cannot gate this. ApplyDirectoryReplacements // skips any chunk that is too short on its own. - changed |= ApplyDirectoryReplacements(chunks, pairs); + if (ApplyDirectoryReplacements(chunks, pairs) is { } replaced) + { + chunks = replaced; + changed = true; + } } } } @@ -148,47 +160,68 @@ public static void RunToBuilder( return null; } - static bool ApplyDirectoryReplacements(List chunks, List pairs) + // Rebuilds the list, as ApplyInline does. Path replacements are never empty, so + // no join can appear and the result needs no coalescing. + static List? ApplyDirectoryReplacements(List chunks, List pairs) { var shortest = pairs[^1].Find.Length; - var changed = false; - var chunkIndex = 0; - while (chunkIndex < chunks.Count) + List? output = null; + + for (var index = 0; index < chunks.Count; index++) { - var chunk = chunks[chunkIndex]; + var chunk = chunks[index]; if (!chunk.Scannable || chunk.Length < shortest) { - chunkIndex++; + output?.Add(chunk); continue; } - if (!TryFindDirectoryMatch(chunks, chunkIndex, pairs, shortest, out var matchStart, out var matchLength, out var replacement)) + var after = index + 1 < chunks.Count ? chunks[index + 1].First : (char?) null; + var offset = 0; + while (chunk.Length - offset >= shortest) { - chunkIndex++; - continue; + var span = chunk.Text.AsSpan(chunk.Start + offset, chunk.Length - offset); + var before = PrecedingChar(output, chunks, index); + if (!TryFindDirectoryMatch(span, before, after, pairs, shortest, out var matchStart, out var matchLength, out var replacement)) + { + break; + } + + output ??= CopyUpTo(chunks, index); + if (matchStart > 0) + { + output.Add(new(chunk.Text, chunk.Start + offset, matchStart, scannable: true)); + } + + output.Add(new(replacement, 0, replacement.Length, scannable: false)); + offset += matchStart + matchLength; } - changed = true; - chunkIndex = Splice(chunks, chunkIndex, matchStart, matchLength, replacement); + if (output != null) + { + var remaining = chunk.Length - offset; + if (remaining > 0) + { + output.Add(new(chunk.Text, chunk.Start + offset, remaining, scannable: true)); + } + } } - return changed; + return output; } static bool TryFindDirectoryMatch( - List chunks, - int chunkIndex, + CharSpan span, + char? beforeSegment, + char? afterSegment, List pairs, int shortest, out int matchStart, out int matchLength, out string replacement) { - var span = chunks[chunkIndex].Span; var anchors = DirectoryReplacements.ItemAnchors.AsSpan(); - var beforeSegment = chunkIndex > 0 ? chunks[chunkIndex - 1].Last : (char?) null; - var afterSegment = chunkIndex + 1 < chunks.Count ? chunks[chunkIndex + 1].First : (char?) null; for (var position = 0; position + shortest <= span.Length; position++) { // Skip to the next candidate first char @@ -255,43 +288,106 @@ static string AssembleString(List chunks) #endif } - static bool ApplyInline( + // Rebuilds the list instead of splicing in place. An in place remove and insert + // shifts every element after the match, so a pass over a list that an earlier + // scrubber already fragmented would cost O(matches x chunks). + // Returns null when nothing matched, leaving the caller's list untouched and + // unallocated. + static List? ApplyInline( List chunks, Scrubber scrubber, Counter counter, IReadOnlyDictionary context) { - var changed = false; - var deleted = false; var effectiveMin = Math.Max(1, scrubber.MinLength); - var chunkIndex = 0; - while (chunkIndex < chunks.Count) + List? output = null; + var deleted = false; + + for (var index = 0; index < chunks.Count; index++) { - var chunk = chunks[chunkIndex]; + var chunk = chunks[index]; if (!chunk.Scannable || chunk.Length < effectiveMin) { - chunkIndex++; + output?.Add(chunk); continue; } - if (!TryFindMatch(chunks, chunkIndex, scrubber, counter, context, out var matchStart, out var matchLength, out var replacement)) + // Chunks past this one are untouched, so the following neighbor is fixed + var after = index + 1 < chunks.Count ? chunks[index + 1].First : (char?) null; + var offset = 0; + while (chunk.Length - offset >= effectiveMin) { - chunkIndex++; - continue; + var span = chunk.Text.AsSpan(chunk.Start + offset, chunk.Length - offset); + var before = PrecedingChar(output, chunks, index); + if (!TryFindMatch(span, before, after, scrubber, counter, context, out var matchStart, out var matchLength, out var replacement)) + { + break; + } + + output ??= CopyUpTo(chunks, index); + if (matchStart > 0) + { + output.Add(new(chunk.Text, chunk.Start + offset, matchStart, scannable: true)); + } + + if (replacement.Length > 0) + { + output.Add(new(replacement, 0, replacement.Length, scannable: false)); + } + else + { + deleted = true; + } + + offset += matchStart + matchLength; } - changed = true; - deleted |= replacement.Length == 0; - chunkIndex = Splice(chunks, chunkIndex, matchStart, matchLength, replacement); + if (output != null) + { + var remaining = chunk.Length - offset; + if (remaining > 0) + { + output.Add(new(chunk.Text, chunk.Start + offset, remaining, scannable: true)); + } + } + } + + if (output == null) + { + return null; } if (deleted) { - CoalesceScannable(chunks); + CoalesceScannable(output); + } + + return output; + } + + // The chunks before index are emitted unchanged, so they seed the rebuilt list + static List CopyUpTo(List chunks, int index) + { + var output = new List(chunks.Count + 4); + for (var copied = 0; copied < index; copied++) + { + output.Add(chunks[copied]); } - return changed; + return output; + } + + // The char before the position being scanned: the last one emitted so far, or + // the end of the preceding chunk while nothing has been emitted + static char? PrecedingChar(List? output, List chunks, int index) + { + if (output != null) + { + return output.Count > 0 ? output[^1].Last : null; + } + + return index > 0 ? chunks[index - 1].Last : null; } // An empty replacement quarantines nothing, so the document text on either @@ -360,39 +456,10 @@ static Chunk Merge(List chunks, int start, int end) return new(builder.ToString(), 0, total, scannable: true); } - // Splits the chunk at chunkIndex into [prefix][replacement][suffix]. - // Returns the index of the suffix chunk (to continue scanning), or the index - // after the inserted chunks when the match ended at the chunk end. - static int Splice(List chunks, int chunkIndex, int matchStart, int matchLength, string replacement) - { - var chunk = chunks[chunkIndex]; - chunks.RemoveAt(chunkIndex); - var insertIndex = chunkIndex; - if (matchStart > 0) - { - chunks.Insert(insertIndex, new(chunk.Text, chunk.Start, matchStart, scannable: true)); - insertIndex++; - } - - if (replacement.Length > 0) - { - chunks.Insert(insertIndex, new(replacement, 0, replacement.Length, scannable: false)); - insertIndex++; - } - - var suffixStart = matchStart + matchLength; - var suffixLength = chunk.Length - suffixStart; - if (suffixLength > 0) - { - chunks.Insert(insertIndex, new(chunk.Text, chunk.Start + suffixStart, suffixLength, scannable: true)); - } - - return insertIndex; - } - static bool TryFindMatch( - List chunks, - int chunkIndex, + CharSpan span, + char? before, + char? after, Scrubber scrubber, Counter counter, IReadOnlyDictionary context, @@ -403,25 +470,25 @@ static bool TryFindMatch( switch (scrubber.Kind) { case ScrubberKind.Replace: - return TryFindReplaceMatch(chunks, chunkIndex, scrubber, out matchStart, out matchLength, out replacement); + return TryFindReplaceMatch(span, before, after, scrubber, out matchStart, out matchLength, out replacement); case ScrubberKind.Window: - return TryFindWindowMatch(chunks, chunkIndex, scrubber, counter, context, out matchStart, out matchLength, out replacement); + return TryFindWindowMatch(span, before, after, scrubber, counter, context, out matchStart, out matchLength, out replacement); case ScrubberKind.Match: - return TryFindCustomMatch(chunks, chunkIndex, scrubber, counter, context, out matchStart, out matchLength, out replacement); + return TryFindCustomMatch(span, scrubber, counter, context, out matchStart, out matchLength, out replacement); default: throw new($"Unexpected inline scrubber kind: {scrubber.Kind}"); } } static bool TryFindReplaceMatch( - List chunks, - int chunkIndex, + CharSpan span, + char? before, + char? after, Scrubber scrubber, out int matchStart, out int matchLength, out string replacement) { - var span = chunks[chunkIndex].Span; var pairs = scrubber.Pairs!; matchStart = -1; matchLength = 0; @@ -453,7 +520,7 @@ static bool TryFindReplaceMatch( } if (scrubber.RequireWordBoundary && - !BoundaryOk(chunks, chunkIndex, index, find.Length)) + !BoundaryOk(span, before, after, index, find.Length)) { searchFrom = index + 1; continue; @@ -470,8 +537,9 @@ static bool TryFindReplaceMatch( } static bool TryFindWindowMatch( - List chunks, - int chunkIndex, + CharSpan span, + char? before, + char? after, Scrubber scrubber, Counter counter, IReadOnlyDictionary context, @@ -479,7 +547,6 @@ static bool TryFindWindowMatch( out int matchLength, out string replacement) { - var span = chunks[chunkIndex].Span; var min = scrubber.MinLength; var max = scrubber.MaxLength!.Value; var matcher = scrubber.WindowMatcher!; @@ -505,8 +572,8 @@ static bool TryFindWindowMatch( } if (scrubber.RequireWordBoundary && - PreviousChar(chunks, chunkIndex, position) is { } before && - char.IsLetterOrDigit(before)) + PreviousChar(span, before, position) is { } precedingChar && + char.IsLetterOrDigit(precedingChar)) { continue; } @@ -515,8 +582,8 @@ static bool TryFindWindowMatch( for (var length = upper; length >= min; length--) { if (scrubber.RequireWordBoundary && - NextChar(chunks, chunkIndex, position + length) is { } after && - char.IsLetterOrDigit(after)) + NextChar(span, after, position + length) is { } followingChar && + char.IsLetterOrDigit(followingChar)) { continue; } @@ -590,8 +657,7 @@ static int NextAnchoredPosition(CharSpan span, int position, int regionEnd, Scru } static bool TryFindCustomMatch( - List chunks, - int chunkIndex, + CharSpan span, Scrubber scrubber, Counter counter, IReadOnlyDictionary context, @@ -599,7 +665,6 @@ static bool TryFindCustomMatch( out int matchLength, out string replacement) { - var span = chunks[chunkIndex].Span; if (!scrubber.SegmentMatcher!(span, counter, context, out matchStart, out matchLength, out var result)) { replacement = string.Empty; @@ -627,16 +692,16 @@ static bool TryFindCustomMatch( return true; } - static bool BoundaryOk(List chunks, int chunkIndex, int index, int length) + static bool BoundaryOk(CharSpan span, char? before, char? after, int index, int length) { - if (PreviousChar(chunks, chunkIndex, index) is { } before && - char.IsLetterOrDigit(before)) + if (PreviousChar(span, before, index) is { } precedingChar && + char.IsLetterOrDigit(precedingChar)) { return false; } - if (NextChar(chunks, chunkIndex, index + length) is { } after && - char.IsLetterOrDigit(after)) + if (NextChar(span, after, index + length) is { } followingChar && + char.IsLetterOrDigit(followingChar)) { return false; } @@ -644,39 +709,27 @@ static bool BoundaryOk(List chunks, int chunkIndex, int index, int length return true; } - // The char before the given position within the chunk. At the chunk start the - // neighbor is the last char of the previous chunk (replacement text counts). - static char? PreviousChar(List chunks, int chunkIndex, int position) + // The char before the given position within the span. At the span start the + // neighbor is the one preceding it in the document (replacement text counts). + static char? PreviousChar(CharSpan span, char? before, int position) { - var chunk = chunks[chunkIndex]; if (position > 0) { - return chunk.Text[chunk.Start + position - 1]; - } - - if (chunkIndex > 0) - { - return chunks[chunkIndex - 1].Last; + return span[position - 1]; } - return null; + return before; } - // The char at the given position within the chunk. At the chunk end the - // neighbor is the first char of the next chunk (replacement text counts). - static char? NextChar(List chunks, int chunkIndex, int position) + // The char at the given position within the span. At the span end the neighbor + // is the one following it in the document (replacement text counts). + static char? NextChar(CharSpan span, char? after, int position) { - var chunk = chunks[chunkIndex]; - if (position < chunk.Length) + if (position < span.Length) { - return chunk.Text[chunk.Start + position]; + return span[position]; } - if (chunkIndex + 1 < chunks.Count) - { - return chunks[chunkIndex + 1].First; - } - - return null; + return after; } } From 8ca08e7138f55839ec86e742cd5a22e12048993d Mon Sep 17 00:00:00 2001 From: Simon Cropp Date: Sun, 19 Jul 2026 19:23:20 +1000 Subject: [PATCH 25/33] . --- src/Benchmarks/NoMatchAllocationBenchmarks.cs | 81 +++++++++++++++++++ .../Serialization/Scrubbers/ScrubEngine.cs | 69 +++++++--------- .../Scrubbers/ScrubEngine_Lines.cs | 27 ++++--- 3 files changed, 128 insertions(+), 49 deletions(-) create mode 100644 src/Benchmarks/NoMatchAllocationBenchmarks.cs diff --git a/src/Benchmarks/NoMatchAllocationBenchmarks.cs b/src/Benchmarks/NoMatchAllocationBenchmarks.cs new file mode 100644 index 0000000000..6116805c66 --- /dev/null +++ b/src/Benchmarks/NoMatchAllocationBenchmarks.cs @@ -0,0 +1,81 @@ +// The no-match pass is the dominant real case: scrubbers are registered globally, +// but most documents (and most serialized property values) contain nothing they +// match. Those passes should allocate nothing. +// Inline_* registers a Replace whose find never occurs. Line_* registers a +// RemoveLinesContaining whose needle never occurs, which routes through the line +// phase. Both_* registers both. +// Allocated is the metric that matters here, not Mean. +[MemoryDiagnoser] +[SimpleJob(iterationCount: 10, warmupCount: 3)] +public class NoMatchAllocationBenchmarks +{ + static Dictionary emptyContext = []; + + string small = null!; + string medium = null!; + string large = null!; + + EngineScrubberSet inlineSet = null!; + EngineScrubberSet lineSet = null!; + EngineScrubberSet bothSet = null!; + + // One Counter for the whole run: Counter.Start allocates several dictionaries, + // which would swamp the engine allocation being measured here. A verify creates + // one Counter and scrubs many values through it. + Counter counter = null!; + + [GlobalCleanup] + public void Cleanup() => counter.Dispose(); + + [GlobalSetup] + public void Setup() + { + counter = Counter.Start(); + small = Build(260); // p50 + medium = Build(2_900); // p90 + large = Build(31_000); // p99 + + inlineSet = EngineScrubberSet.ForScrubbers([Scrubber.Replace("NEVER_OCCURS_TOKEN", "{X}")]); + lineSet = EngineScrubberSet.ForScrubbers([Scrubber.RemoveLinesContaining(StringComparison.Ordinal, "NEVER_OCCURS_TOKEN")]); + bothSet = EngineScrubberSet.ForScrubbers( + [ + Scrubber.Replace("NEVER_OCCURS_TOKEN", "{X}"), + Scrubber.RemoveLinesContaining(StringComparison.Ordinal, "ALSO_NEVER") + ]); + } + + static string Build(int targetChars) + { + var builder = new StringBuilder(); + while (builder.Length < targetChars) + { + builder.Append(" \"name\": \"some value here\",\n"); + } + + return builder.ToString(); + } + + string Run(string input, EngineScrubberSet set) => + ScrubEngine.Run(input, set, counter, emptyContext, applyDirectoryReplacements: false); + + [Benchmark] + public string Inline_Small() => Run(small, inlineSet); + + [Benchmark] + public string Inline_Medium() => Run(medium, inlineSet); + + [Benchmark] + public string Inline_Large() => Run(large, inlineSet); + + [Benchmark] + public string Line_Small() => Run(small, lineSet); + + [Benchmark] + public string Line_Medium() => Run(medium, lineSet); + + [Benchmark] + public string Line_Large() => Run(large, lineSet); + + [Benchmark] + public string Both_Large() => Run(large, bothSet); +} diff --git a/src/Verify/Serialization/Scrubbers/ScrubEngine.cs b/src/Verify/Serialization/Scrubbers/ScrubEngine.cs index cb01ed25e7..74a7370a29 100644 --- a/src/Verify/Serialization/Scrubbers/ScrubEngine.cs +++ b/src/Verify/Serialization/Scrubbers/ScrubEngine.cs @@ -111,8 +111,9 @@ public static void RunToBuilder( // the instance built for the culture in effect for this scrub var scrubber = registered.Resolve(); - chunks ??= [new(source, 0, source.Length, scannable: true)]; - if (ApplyInline(chunks, scrubber, counter, context) is { } applied) + // chunks stays null until something matches, so a pass that changes + // nothing allocates nothing + if (ApplyInline(chunks, source, scrubber, counter, context) is { } applied) { chunks = applied; changed = true; @@ -123,31 +124,18 @@ public static void RunToBuilder( if (applyDirectoryReplacements) { var pairs = DirectoryReplacements.Items; - if (pairs.Count > 0) + // When nothing has run the source is still the whole document, so its + // length gates the scan. Once a scrubber has run the document may have + // grown past the shortest find, and ApplyDirectoryReplacements skips any + // chunk that is too short on its own. + if (pairs.Count > 0 && + (chunks != null || + source.Length >= DirectoryReplacements.ShortestFindLength)) { - if (chunks == null) + if (ApplyDirectoryReplacements(chunks, source, pairs) is { } replaced) { - // Nothing has run, so the source is still the whole document - if (source.Length >= DirectoryReplacements.ShortestFindLength) - { - chunks = [new(source, 0, source.Length, scannable: true)]; - if (ApplyDirectoryReplacements(chunks, pairs) is { } replaced) - { - chunks = replaced; - changed = true; - } - } - } - else - { - // A scrubber may have grown the document past the shortest find, - // so the pre-scrub length cannot gate this. ApplyDirectoryReplacements - // skips any chunk that is too short on its own. - if (ApplyDirectoryReplacements(chunks, pairs) is { } replaced) - { - chunks = replaced; - changed = true; - } + chunks = replaced; + changed = true; } } } @@ -162,14 +150,15 @@ public static void RunToBuilder( // Rebuilds the list, as ApplyInline does. Path replacements are never empty, so // no join can appear and the result needs no coalescing. - static List? ApplyDirectoryReplacements(List chunks, List pairs) + static List? ApplyDirectoryReplacements(List? chunks, string source, List pairs) { var shortest = pairs[^1].Find.Length; List? output = null; + var count = chunks?.Count ?? 1; - for (var index = 0; index < chunks.Count; index++) + for (var index = 0; index < count; index++) { - var chunk = chunks[index]; + var chunk = chunks?[index] ?? new(source, 0, source.Length, scannable: true); if (!chunk.Scannable || chunk.Length < shortest) { @@ -177,7 +166,7 @@ public static void RunToBuilder( continue; } - var after = index + 1 < chunks.Count ? chunks[index + 1].First : (char?) null; + var after = chunks != null && index + 1 < count ? chunks[index + 1].First : (char?) null; var offset = 0; while (chunk.Length - offset >= shortest) { @@ -294,7 +283,8 @@ static string AssembleString(List chunks) // Returns null when nothing matched, leaving the caller's list untouched and // unallocated. static List? ApplyInline( - List chunks, + List? chunks, + string source, Scrubber scrubber, Counter counter, IReadOnlyDictionary context) @@ -302,10 +292,13 @@ static string AssembleString(List chunks) var effectiveMin = Math.Max(1, scrubber.MinLength); List? output = null; var deleted = false; + var count = chunks?.Count ?? 1; - for (var index = 0; index < chunks.Count; index++) + for (var index = 0; index < count; index++) { - var chunk = chunks[index]; + // A null list means nothing has changed yet, so the document is still + // the whole source + var chunk = chunks?[index] ?? new(source, 0, source.Length, scannable: true); if (!chunk.Scannable || chunk.Length < effectiveMin) { @@ -314,7 +307,7 @@ static string AssembleString(List chunks) } // Chunks past this one are untouched, so the following neighbor is fixed - var after = index + 1 < chunks.Count ? chunks[index + 1].First : (char?) null; + var after = chunks != null && index + 1 < count ? chunks[index + 1].First : (char?) null; var offset = 0; while (chunk.Length - offset >= effectiveMin) { @@ -367,12 +360,12 @@ static string AssembleString(List chunks) } // The chunks before index are emitted unchanged, so they seed the rebuilt list - static List CopyUpTo(List chunks, int index) + static List CopyUpTo(List? chunks, int index) { - var output = new List(chunks.Count + 4); + var output = new List((chunks?.Count ?? 1) + 4); for (var copied = 0; copied < index; copied++) { - output.Add(chunks[copied]); + output.Add(chunks![copied]); } return output; @@ -380,14 +373,14 @@ static List CopyUpTo(List chunks, int index) // The char before the position being scanned: the last one emitted so far, or // the end of the preceding chunk while nothing has been emitted - static char? PrecedingChar(List? output, List chunks, int index) + static char? PrecedingChar(List? output, List? chunks, int index) { if (output != null) { return output.Count > 0 ? output[^1].Last : null; } - return index > 0 ? chunks[index - 1].Last : null; + return chunks != null && index > 0 ? chunks[index - 1].Last : null; } // An empty replacement quarantines nothing, so the document text on either diff --git a/src/Verify/Serialization/Scrubbers/ScrubEngine_Lines.cs b/src/Verify/Serialization/Scrubbers/ScrubEngine_Lines.cs index 76b229b19f..1c4ca1eeab 100644 --- a/src/Verify/Serialization/Scrubbers/ScrubEngine_Lines.cs +++ b/src/Verify/Serialization/Scrubbers/ScrubEngine_Lines.cs @@ -25,7 +25,9 @@ readonly struct LineItem(int start, int end, string? fresh) static List? LinePhase(string source, EngineScrubberSet set) { var endsWithNewline = source[^1] == '\n'; - var items = new List(); + // Created on the first change, so a document no line scrubber touches + // allocates nothing + List? items = null; var changed = false; // Coalesce runs of adjacent untouched kept lines (including their inner @@ -37,7 +39,7 @@ void FlushRun() { if (runStart is { } start) { - items.Add(new(start, runEnd, null)); + (items ??= []).Add(new(start, runEnd, null)); runStart = null; } } @@ -70,7 +72,7 @@ void FlushRun() { changed = true; FlushRun(); - items.Add(new(0, 0, current)); + (items ??= []).Add(new(0, 0, current)); } else { @@ -87,12 +89,12 @@ void FlushRun() position = newlineIndex + 1; } - FlushRun(); - - // RemoveEmptyLines trims the trailing newline even when no line was dropped + // RemoveEmptyLines trims the trailing newline even when no line was dropped. + // Checked before the final flush so an untouched document stays unallocated: + // flushing would produce an item exactly when one exists or a run is pending. if (set.TrimOuterEmptyLines && endsWithNewline && - items.Count > 0) + (items != null || runStart != null)) { changed = true; } @@ -102,11 +104,14 @@ void FlushRun() return null; } - var chunks = new List(items.Count + 1); - for (var index = 0; index < items.Count; index++) + FlushRun(); + + var itemCount = items?.Count ?? 0; + var chunks = new List(itemCount + 1); + for (var index = 0; index < itemCount; index++) { - var item = items[index]; - var needsSeparator = index < items.Count - 1 || + var item = items![index]; + var needsSeparator = index < itemCount - 1 || (endsWithNewline && !set.TrimOuterEmptyLines); if (item.Fresh is { } fresh) From c903eac3a0f7574c518c9d37efde152e9fdb58cd Mon Sep 17 00:00:00 2001 From: Simon Cropp Date: Sun, 19 Jul 2026 19:33:12 +1000 Subject: [PATCH 26/33] . --- .../LegacyPropertyValueTests.cs | 74 +++++++++++++++++++ .../LegacyScrubberPathBenchmarks.cs | 65 ++++++++++++++++ .../Serialization/Scrubbers/ApplyScrubbers.cs | 12 ++- .../Scrubbers/EngineScrubberSet.cs | 4 + 4 files changed, 151 insertions(+), 4 deletions(-) create mode 100644 src/ApplyScrubbersTests/LegacyPropertyValueTests.cs create mode 100644 src/Benchmarks/LegacyScrubberPathBenchmarks.cs diff --git a/src/ApplyScrubbersTests/LegacyPropertyValueTests.cs b/src/ApplyScrubbersTests/LegacyPropertyValueTests.cs new file mode 100644 index 0000000000..36977ddfef --- /dev/null +++ b/src/ApplyScrubbersTests/LegacyPropertyValueTests.cs @@ -0,0 +1,74 @@ +// The property value path when a legacy StringBuilder scrubber is registered: +// span scrubbers run, then the legacy pass, then path replacement and the newline +// fix. Covers ApplyScrubbers.ApplyWithLegacy, which ApplyForExtension does not. +public class LegacyPropertyValueTests +{ + static LegacyPropertyValueTests() => + EngineRunner.UseFakeDirectories(); + + static string Run(string input, Action configure) + { + var settings = new VerifySettings(); + configure(settings); + using var counter = Counter.Start(); + return ApplyScrubbers.ApplyForPropertyValue(input, settings, counter); + } + + [Fact] + public void LegacyRunsAfterSpanScrubbers() + { + var result = Run( + "a", + settings => + { + settings.AddScrubber(Scrubber.Replace("a", "1")); + settings.AddScrubber(builder => builder.Replace("1", "2")); + }); + Assert.Equal("2", result); + } + + [Fact] + public void PathWrittenByLegacyIsScrubbed() + { + var result = Run( + "value ", + settings => settings.AddScrubber(builder => builder.Append("C:/Code/TheSolution/TheProject/file.txt"))); + Assert.Equal("value {ProjectDirectory}file.txt", result); + } + + [Fact] + public void LegacySeesRawPath() + { + var result = Run( + "C:/Code/TheSolution/TheProject/data", + settings => settings.AddScrubber(builder => builder.Replace("C:/Code/TheSolution/TheProject/data", "{custom}"))); + Assert.Equal("{custom}", result); + } + + [Fact] + public void NewlinesInjectedByLegacyAreNormalized() + { + var result = Run( + "value", + settings => settings.AddScrubber(builder => builder.Append("\r\nline\rmore"))); + Assert.Equal("value\nline\nmore", result); + } + + [Fact] + public void PathAdjacentToInjectedCarriageReturnIsStillScrubbed() + { + // Newline normalization now runs before path replacement rather than after, + // so a path sitting next to an injected '\r' must still be replaced + var result = Run( + "value", + settings => settings.AddScrubber(builder => builder.Append("\r\nC:/Code/TheSolution/TheProject/f.txt\r\n"))); + Assert.Equal("value\n{ProjectDirectory}f.txt\n", result); + } + + [Fact] + public void UnchangedValueIsReturnedAsIs() + { + var result = Run("plain value", settings => settings.AddScrubber(_ => { })); + Assert.Equal("plain value", result); + } +} diff --git a/src/Benchmarks/LegacyScrubberPathBenchmarks.cs b/src/Benchmarks/LegacyScrubberPathBenchmarks.cs new file mode 100644 index 0000000000..26fa911efc --- /dev/null +++ b/src/Benchmarks/LegacyScrubberPathBenchmarks.cs @@ -0,0 +1,65 @@ +// When any legacy AddScrubber(Action) is registered, the span engine +// runs first and a StringBuilder pass follows. The trailing path replacement and +// newline fix then run over that builder. +// PropertyValue_* is the hot one: it runs once per serialized string value, and it +// materialized the builder twice, once inside the path pass and once to return. +// Document_* runs once per file and materializes once. +[MemoryDiagnoser] +[SimpleJob(iterationCount: 10, warmupCount: 3)] +public class LegacyScrubberPathBenchmarks +{ + string small = null!; + string medium = null!; + string large = null!; + + VerifySettings settings = null!; + Counter counter = null!; + + [GlobalSetup] + public void Setup() + { + // Deterministic path replacements, so the trailing pass has real work to scan + DirectoryReplacements.UseAssembly("C:/Code/TheSolution", "C:/Code/TheSolution/TheProject"); + + small = Build(260); + medium = Build(2_900); + large = Build(31_000); + + settings = new(); + // A no-op legacy scrubber, enough to select the legacy pipeline + settings.AddScrubber(_ => { }); + + counter = Counter.Start(); + } + + [GlobalCleanup] + public void Cleanup() => counter.Dispose(); + + static string Build(int targetChars) + { + var builder = new StringBuilder(); + while (builder.Length < targetChars) + { + builder.Append(" \"name\": \"some value here\",\n"); + } + + return builder.ToString(); + } + + [Benchmark] + public string PropertyValue_Small() => ApplyScrubbers.ApplyForPropertyValue(small, settings, counter); + + [Benchmark] + public string PropertyValue_Medium() => ApplyScrubbers.ApplyForPropertyValue(medium, settings, counter); + + [Benchmark] + public string PropertyValue_Large() => ApplyScrubbers.ApplyForPropertyValue(large, settings, counter); + + [Benchmark] + public int Document_Large() + { + var builder = new StringBuilder(large); + ApplyScrubbers.ApplyForExtension("txt", builder, settings, counter); + return builder.Length; + } +} diff --git a/src/Verify/Serialization/Scrubbers/ApplyScrubbers.cs b/src/Verify/Serialization/Scrubbers/ApplyScrubbers.cs index 5f94469b41..4f94b49c57 100644 --- a/src/Verify/Serialization/Scrubbers/ApplyScrubbers.cs +++ b/src/Verify/Serialization/Scrubbers/ApplyScrubbers.cs @@ -136,10 +136,14 @@ static string ApplyWithLegacy(string value, VerifySettings settings, Counter cou scrubber(builder, counter, settings.Context); } - DirectoryReplacements.Replace(builder); - - builder.FixNewlines(); - return builder.ToString(); + // Path replacements and the newline fix both run on the engine, so the + // builder is materialized once. Doing them on the builder materialized it + // again inside the path scan, copying the whole value twice. + // The engine normalizes newlines before replacing paths rather than after, + // which cannot change a match: a path never contains '\r' or '\n', and both + // are non alphanumeric so the boundary and trailing separator rules see them + // the same either way. + return ScrubEngine.Run(builder.ToString(), EngineScrubberSet.Empty, counter, settings.Context, applyDirectoryReplacements: true); } static bool HasLegacyForExtension(VerifySettings settings, string extension) => diff --git a/src/Verify/Serialization/Scrubbers/EngineScrubberSet.cs b/src/Verify/Serialization/Scrubbers/EngineScrubberSet.cs index 7d8650594f..4793709421 100644 --- a/src/Verify/Serialization/Scrubbers/EngineScrubberSet.cs +++ b/src/Verify/Serialization/Scrubbers/EngineScrubberSet.cs @@ -24,6 +24,10 @@ sealed class EngineScrubberSet static EngineScrubberSet empty = new([], [], []); + // A set with no scrubbers, for a pass that only needs the engine's pinned + // trailing work (path replacements and newline normalization) + public static EngineScrubberSet Empty => empty; + EngineScrubberSet(Scrubber[] lineDrops, Scrubber[] lineTransforms, Scrubber[] inline) { LineDrops = lineDrops; From b23a58a7ec8312dcc459ab99a0ef8b1595d8004f Mon Sep 17 00:00:00 2001 From: Simon Cropp Date: Sun, 19 Jul 2026 19:47:32 +1000 Subject: [PATCH 27/33] . --- .../DirectorySnapshotTests.cs | 44 +++++++++++++++++++ .../Scrubbers/DirectoryReplacements.cs | 43 +++++++++++------- .../Serialization/Scrubbers/ScrubEngine.cs | 21 ++++----- 3 files changed, 83 insertions(+), 25 deletions(-) create mode 100644 src/ApplyScrubbersTests/DirectorySnapshotTests.cs diff --git a/src/ApplyScrubbersTests/DirectorySnapshotTests.cs b/src/ApplyScrubbersTests/DirectorySnapshotTests.cs new file mode 100644 index 0000000000..5913ec5dcb --- /dev/null +++ b/src/ApplyScrubbersTests/DirectorySnapshotTests.cs @@ -0,0 +1,44 @@ +// The anchors and the shortest find are derived from the pairs, so UseAssembly has +// to keep them in step. A scan that used anchors not covering a pair's first char +// would never land on that path. +public class DirectorySnapshotTests +{ + static DirectorySnapshotTests() => + EngineRunner.UseFakeDirectories(); + + [Fact] + public void AnchorsCoverEveryPair() + { + var snapshot = DirectoryReplacements.Current; + Assert.NotEmpty(snapshot.Pairs); + + foreach (var pair in snapshot.Pairs) + { + Assert.Contains(pair.Find[0], snapshot.Anchors); + } + } + + [Fact] + public void ShortestFindLengthMatchesThePairs() + { + var snapshot = DirectoryReplacements.Current; + var shortest = int.MaxValue; + foreach (var pair in snapshot.Pairs) + { + shortest = Math.Min(shortest, pair.Find.Length); + } + + Assert.Equal(shortest, snapshot.ShortestFindLength); + } + + [Fact] + public void PairsAreOrderedLongestFirst() + { + // The most specific path has to win at any given position + var snapshot = DirectoryReplacements.Current; + for (var index = 1; index < snapshot.Pairs.Count; index++) + { + Assert.True(snapshot.Pairs[index - 1].Find.Length >= snapshot.Pairs[index].Find.Length); + } + } +} diff --git a/src/Verify/Serialization/Scrubbers/DirectoryReplacements.cs b/src/Verify/Serialization/Scrubbers/DirectoryReplacements.cs index b99962f39e..7c5a19dcc1 100644 --- a/src/Verify/Serialization/Scrubbers/DirectoryReplacements.cs +++ b/src/Verify/Serialization/Scrubbers/DirectoryReplacements.cs @@ -20,19 +20,27 @@ public Pair(string find, string replace) public string Replace { get; } } - static List items = []; - static int shortestFindLength = int.MaxValue; - static string itemAnchors = ""; + // The anchors and the shortest find are derived from the pairs, so the three + // travel together rather than as separate statics a scan has to re-read. + // Assigned once, from AssignTargetAssembly. + internal sealed class Snapshot(List pairs, int shortestFindLength, string anchors) + { + public readonly List Pairs = pairs; - // Length of the shortest Find, so callers can cheaply skip values that are - // too short to contain any replacement. int.MaxValue when there are none. - public static int ShortestFindLength => shortestFindLength; + public readonly int ShortestFindLength = shortestFindLength; + + // The distinct first chars of the Finds, so scans can skip to candidate + // positions with a vectorized IndexOfAny instead of probing every position + public readonly string Anchors = anchors; + } - internal static List Items => items; + static Snapshot current = new([], int.MaxValue, ""); - // The distinct first chars of the Finds, so scans can skip to candidate - // positions with a vectorized IndexOfAny instead of probing every position - internal static string ItemAnchors => itemAnchors; + internal static Snapshot Current => current; + + // Length of the shortest Find, so callers can cheaply skip values that are + // too short to contain any replacement. int.MaxValue when there are none. + public static int ShortestFindLength => current.ShortestFindLength; internal static string BuildAnchors(List pairs) { @@ -56,8 +64,11 @@ internal static string BuildAnchors(List pairs) return new([.. anchors]); } - public static void Replace(StringBuilder builder) => - Replace(builder, items, itemAnchors); + public static void Replace(StringBuilder builder) + { + var snapshot = current; + Replace(builder, snapshot.Pairs, snapshot.Anchors); + } public static void Replace(StringBuilder builder, List pairs) => Replace(builder, pairs, BuildAnchors(pairs)); @@ -166,11 +177,13 @@ public static void UseAssembly(string? solutionDir, string projectDir) } AddProjectAndSolutionReplacements(solutionDir, projectDir, values); - items = values + var ordered = values .OrderByDescending(_ => _.Find.Length) .ToList(); - shortestFindLength = items.Count == 0 ? int.MaxValue : items[^1].Find.Length; - itemAnchors = BuildAnchors(items); + current = new( + ordered, + ordered.Count == 0 ? int.MaxValue : ordered[^1].Find.Length, + BuildAnchors(ordered)); } static void AddProjectAndSolutionReplacements(string? solutionDir, string projectDir, List replacements) diff --git a/src/Verify/Serialization/Scrubbers/ScrubEngine.cs b/src/Verify/Serialization/Scrubbers/ScrubEngine.cs index 74a7370a29..5bc31e2d9c 100644 --- a/src/Verify/Serialization/Scrubbers/ScrubEngine.cs +++ b/src/Verify/Serialization/Scrubbers/ScrubEngine.cs @@ -123,16 +123,16 @@ public static void RunToBuilder( // Path replacements are pinned last so user scrubbers always see raw paths if (applyDirectoryReplacements) { - var pairs = DirectoryReplacements.Items; + var replacements = DirectoryReplacements.Current; // When nothing has run the source is still the whole document, so its // length gates the scan. Once a scrubber has run the document may have // grown past the shortest find, and ApplyDirectoryReplacements skips any // chunk that is too short on its own. - if (pairs.Count > 0 && + if (replacements.Pairs.Count > 0 && (chunks != null || - source.Length >= DirectoryReplacements.ShortestFindLength)) + source.Length >= replacements.ShortestFindLength)) { - if (ApplyDirectoryReplacements(chunks, source, pairs) is { } replaced) + if (ApplyDirectoryReplacements(chunks, source, replacements) is { } replaced) { chunks = replaced; changed = true; @@ -150,9 +150,9 @@ public static void RunToBuilder( // Rebuilds the list, as ApplyInline does. Path replacements are never empty, so // no join can appear and the result needs no coalescing. - static List? ApplyDirectoryReplacements(List? chunks, string source, List pairs) + static List? ApplyDirectoryReplacements(List? chunks, string source, DirectoryReplacements.Snapshot replacements) { - var shortest = pairs[^1].Find.Length; + var shortest = replacements.ShortestFindLength; List? output = null; var count = chunks?.Count ?? 1; @@ -172,7 +172,7 @@ public static void RunToBuilder( { var span = chunk.Text.AsSpan(chunk.Start + offset, chunk.Length - offset); var before = PrecedingChar(output, chunks, index); - if (!TryFindDirectoryMatch(span, before, after, pairs, shortest, out var matchStart, out var matchLength, out var replacement)) + if (!TryFindDirectoryMatch(span, before, after, replacements, out var matchStart, out var matchLength, out var replacement)) { break; } @@ -204,13 +204,14 @@ static bool TryFindDirectoryMatch( CharSpan span, char? beforeSegment, char? afterSegment, - List pairs, - int shortest, + DirectoryReplacements.Snapshot replacements, out int matchStart, out int matchLength, out string replacement) { - var anchors = DirectoryReplacements.ItemAnchors.AsSpan(); + var pairs = replacements.Pairs; + var shortest = replacements.ShortestFindLength; + var anchors = replacements.Anchors.AsSpan(); for (var position = 0; position + shortest <= span.Length; position++) { // Skip to the next candidate first char From 2df9c404170cf87086edd7a4d16206cf2c040048 Mon Sep 17 00:00:00 2001 From: Simon Cropp Date: Sun, 19 Jul 2026 20:01:45 +1000 Subject: [PATCH 28/33] . --- src/ApplyScrubbersTests/LegacyInteropTests.cs | 12 + src/Benchmarks/LegacyDateScrubber.cs | 225 +++++++++++++++++ src/Benchmarks/LegacyScrubbers.cs | 226 ------------------ .../Serialization/Scrubbers/ApplyScrubbers.cs | 7 +- .../Scrubbers/EngineScrubberSet.cs | 5 - .../Scrubbers/ScrubEngine_Lines.cs | 2 - .../Serialization/Scrubbers/Scrubber.cs | 5 - 7 files changed, 242 insertions(+), 240 deletions(-) create mode 100644 src/Benchmarks/LegacyDateScrubber.cs diff --git a/src/ApplyScrubbersTests/LegacyInteropTests.cs b/src/ApplyScrubbersTests/LegacyInteropTests.cs index faa1dc010c..65b897380d 100644 --- a/src/ApplyScrubbersTests/LegacyInteropTests.cs +++ b/src/ApplyScrubbersTests/LegacyInteropTests.cs @@ -41,6 +41,18 @@ public void LegacyLocations_StillHonored() Assert.Equal("value one two", result); } + [Fact] + public void LegacyDoesNotSeeCarriageReturns() + { + // The engine normalizes newlines before the legacy pass, so a legacy + // scrubber matching a literal "\r\n" no longer fires. Declared as a + // breaking change in the 32.0.0 release notes. + var result = RunForExtension( + "a\r\nb", + settings => settings.AddScrubber(builder => builder.Replace("a\r\nb", "MATCHED"))); + Assert.Equal("a\nb", result); + } + [Fact] public void FixNewlines_AfterLegacy() { diff --git a/src/Benchmarks/LegacyDateScrubber.cs b/src/Benchmarks/LegacyDateScrubber.cs new file mode 100644 index 0000000000..ac3dc24bb6 --- /dev/null +++ b/src/Benchmarks/LegacyDateScrubber.cs @@ -0,0 +1,225 @@ +static class LegacyDateScrubber +{ + delegate bool TryConvert( + ReadOnlySpan span, + string format, + Counter counter, + Culture culture, + [NotNullWhen(true)] out string? result); + + static bool TryConvertDate( + ReadOnlySpan span, + string format, + Counter counter, + Culture culture, + [NotNullWhen(true)] out string? result) + { + if (DateOnly.TryParseExact(span, format, culture, DateTimeStyles.None, out var date)) + { + result = counter.Convert(date); + return true; + } + + result = null; + return false; + } + + public static Action BuildDateScrubber(string format, Culture? culture) => + (builder, counter) => ReplaceDates(builder, format, counter, culture ?? Culture.CurrentCulture); + + public static void ReplaceDates(StringBuilder builder, string format, Counter counter, Culture culture) => + ReplaceInner( + builder, + format, + counter, + culture, + TryConvertDate); + + public static Action BuildDateTimeOffsetScrubber(string format, Culture? culture) => + (builder, counter) => ReplaceDateTimeOffsets(builder, format, counter, culture ?? Culture.CurrentCulture); + + static bool TryConvertDateTimeOffset( + ReadOnlySpan span, + string format, + Counter counter, + Culture culture, + [NotNullWhen(true)] out string? result) + { + if (DateTimeOffset.TryParseExact(span, format, culture, DateTimeStyles.None, out var date)) + { + result = counter.Convert(date); + return true; + } + + result = null; + return false; + } + + public static void ReplaceDateTimeOffsets(StringBuilder builder, string format, Counter counter, Culture culture) + { + ReplaceInner( + builder, + format, + counter, + culture, + TryConvertDateTimeOffset); + if (TryGetFormatWithUpperMillisecondsTrimmed(format, out var trimmedFormat)) + { + ReplaceInner( + builder, + trimmedFormat, + counter, + culture, + TryConvertDateTimeOffset); + } + } + + static bool TryConvertDateTime( + ReadOnlySpan span, + string format, + Counter counter, + Culture culture, + [NotNullWhen(true)] out string? result) + { + if (DateTime.TryParseExact(span, format, culture, DateTimeStyles.None, out var date)) + { + result = counter.Convert(date); + return true; + } + + result = null; + return false; + } + + public static Action BuildDateTimeScrubber(string format, Culture? culture) => + (builder, counter) => ReplaceDateTimes(builder, format, counter, culture ?? Culture.CurrentCulture); + + public static void ReplaceDateTimes(StringBuilder builder, string format, Counter counter, Culture culture) + { + ReplaceInner( + builder, + format, + counter, + culture, + TryConvertDateTime); + if (TryGetFormatWithUpperMillisecondsTrimmed(format, out var trimmedFormat)) + { + ReplaceInner( + builder, + trimmedFormat, + counter, + culture, + TryConvertDateTime); + } + } + + static bool TryGetFormatWithUpperMillisecondsTrimmed(string format, [NotNullWhen(true)] out string? trimmedFormat) + { + if (format.EndsWith(".FFFF")) + { + trimmedFormat = format[..^5]; + return true; + } + + if (format.EndsWith(".FFF")) + { + trimmedFormat = format[..^4]; + return true; + } + + if (format.EndsWith(".FF")) + { + trimmedFormat = format[..^3]; + return true; + } + + if (format.EndsWith(".F")) + { + trimmedFormat = format[..^2]; + return true; + } + + trimmedFormat = null; + return false; + } + + static void ReplaceInner(StringBuilder builder, string format, Counter counter, Culture culture, TryConvert tryConvertDate) + { + if (!counter.ScrubDateTimes) + { + return; + } + + var (max, min) = DateFormatLengthCalculator.GetLength(format, culture); + + if (builder.Length < min) + { + return; + } + + if (min == max) + { + ReplaceFixedLength(builder, format, counter, culture, tryConvertDate, max); + + return; + } + + ReplaceVariableLength(builder, format, counter, culture, tryConvertDate, max, min); + } + + static void ReplaceVariableLength(StringBuilder builder, string format, Counter counter, Culture culture, TryConvert tryConvertDate, int longest, int shortest) + { + var value = builder.AsSpan(); + var builderIndex = 0; + for (var index = 0; index <= value.Length; index++) + { + var found = false; + for (var length = longest; length >= shortest; length--) + { + var end = index + length; + if (end > value.Length) + { + continue; + } + + var slice = value.Slice(index, length); + if (tryConvertDate(slice, format, counter, culture, out var convert)) + { + builder.Overwrite(convert, builderIndex, length); + builderIndex += convert.Length; + index += length - 1; + found = true; + break; + } + } + + if (found) + { + continue; + } + + builderIndex++; + } + } + + static void ReplaceFixedLength(StringBuilder builder, string format, Counter counter, Culture culture, TryConvert tryConvertDate, int length) + { + var value = builder.AsSpan(); + var builderIndex = 0; + var increment = length - 1; + for (var index = 0; index <= value.Length - length; index++) + { + var slice = value.Slice(index, length); + if (tryConvertDate(slice, format, counter, culture, out var convert)) + { + builder.Overwrite(convert, builderIndex, length); + builderIndex += convert.Length; + index += increment; + } + else + { + builderIndex++; + } + } + } +} \ No newline at end of file diff --git a/src/Benchmarks/LegacyScrubbers.cs b/src/Benchmarks/LegacyScrubbers.cs index 27385f916d..0d4a7ce9f1 100644 --- a/src/Benchmarks/LegacyScrubbers.cs +++ b/src/Benchmarks/LegacyScrubbers.cs @@ -161,230 +161,4 @@ readonly struct Match(int index, string value) public readonly int Index = index; public readonly string Value = value; } -} - -static class LegacyDateScrubber -{ - delegate bool TryConvert( - ReadOnlySpan span, - string format, - Counter counter, - Culture culture, - [NotNullWhen(true)] out string? result); - - static bool TryConvertDate( - ReadOnlySpan span, - string format, - Counter counter, - Culture culture, - [NotNullWhen(true)] out string? result) - { - if (DateOnly.TryParseExact(span, format, culture, DateTimeStyles.None, out var date)) - { - result = counter.Convert(date); - return true; - } - - result = null; - return false; - } - - public static Action BuildDateScrubber(string format, Culture? culture) => - (builder, counter) => ReplaceDates(builder, format, counter, culture ?? Culture.CurrentCulture); - - public static void ReplaceDates(StringBuilder builder, string format, Counter counter, Culture culture) => - ReplaceInner( - builder, - format, - counter, - culture, - TryConvertDate); - - public static Action BuildDateTimeOffsetScrubber(string format, Culture? culture) => - (builder, counter) => ReplaceDateTimeOffsets(builder, format, counter, culture ?? Culture.CurrentCulture); - - static bool TryConvertDateTimeOffset( - ReadOnlySpan span, - string format, - Counter counter, - Culture culture, - [NotNullWhen(true)] out string? result) - { - if (DateTimeOffset.TryParseExact(span, format, culture, DateTimeStyles.None, out var date)) - { - result = counter.Convert(date); - return true; - } - - result = null; - return false; - } - - public static void ReplaceDateTimeOffsets(StringBuilder builder, string format, Counter counter, Culture culture) - { - ReplaceInner( - builder, - format, - counter, - culture, - TryConvertDateTimeOffset); - if (TryGetFormatWithUpperMillisecondsTrimmed(format, out var trimmedFormat)) - { - ReplaceInner( - builder, - trimmedFormat, - counter, - culture, - TryConvertDateTimeOffset); - } - } - - static bool TryConvertDateTime( - ReadOnlySpan span, - string format, - Counter counter, - Culture culture, - [NotNullWhen(true)] out string? result) - { - if (DateTime.TryParseExact(span, format, culture, DateTimeStyles.None, out var date)) - { - result = counter.Convert(date); - return true; - } - - result = null; - return false; - } - - public static Action BuildDateTimeScrubber(string format, Culture? culture) => - (builder, counter) => ReplaceDateTimes(builder, format, counter, culture ?? Culture.CurrentCulture); - - public static void ReplaceDateTimes(StringBuilder builder, string format, Counter counter, Culture culture) - { - ReplaceInner( - builder, - format, - counter, - culture, - TryConvertDateTime); - if (TryGetFormatWithUpperMillisecondsTrimmed(format, out var trimmedFormat)) - { - ReplaceInner( - builder, - trimmedFormat, - counter, - culture, - TryConvertDateTime); - } - } - - static bool TryGetFormatWithUpperMillisecondsTrimmed(string format, [NotNullWhen(true)] out string? trimmedFormat) - { - if (format.EndsWith(".FFFF")) - { - trimmedFormat = format[..^5]; - return true; - } - - if (format.EndsWith(".FFF")) - { - trimmedFormat = format[..^4]; - return true; - } - - if (format.EndsWith(".FF")) - { - trimmedFormat = format[..^3]; - return true; - } - - if (format.EndsWith(".F")) - { - trimmedFormat = format[..^2]; - return true; - } - - trimmedFormat = null; - return false; - } - - static void ReplaceInner(StringBuilder builder, string format, Counter counter, Culture culture, TryConvert tryConvertDate) - { - if (!counter.ScrubDateTimes) - { - return; - } - - var (max, min) = DateFormatLengthCalculator.GetLength(format, culture); - - if (builder.Length < min) - { - return; - } - - if (min == max) - { - ReplaceFixedLength(builder, format, counter, culture, tryConvertDate, max); - - return; - } - - ReplaceVariableLength(builder, format, counter, culture, tryConvertDate, max, min); - } - - static void ReplaceVariableLength(StringBuilder builder, string format, Counter counter, Culture culture, TryConvert tryConvertDate, int longest, int shortest) - { - var value = builder.AsSpan(); - var builderIndex = 0; - for (var index = 0; index <= value.Length; index++) - { - var found = false; - for (var length = longest; length >= shortest; length--) - { - var end = index + length; - if (end > value.Length) - { - continue; - } - - var slice = value.Slice(index, length); - if (tryConvertDate(slice, format, counter, culture, out var convert)) - { - builder.Overwrite(convert, builderIndex, length); - builderIndex += convert.Length; - index += length - 1; - found = true; - break; - } - } - - if (found) - { - continue; - } - - builderIndex++; - } - } - - static void ReplaceFixedLength(StringBuilder builder, string format, Counter counter, Culture culture, TryConvert tryConvertDate, int length) - { - var value = builder.AsSpan(); - var builderIndex = 0; - var increment = length - 1; - for (var index = 0; index <= value.Length - length; index++) - { - var slice = value.Slice(index, length); - if (tryConvertDate(slice, format, counter, culture, out var convert)) - { - builder.Overwrite(convert, builderIndex, length); - builderIndex += convert.Length; - index += increment; - } - else - { - builderIndex++; - } - } - } } \ No newline at end of file diff --git a/src/Verify/Serialization/Scrubbers/ApplyScrubbers.cs b/src/Verify/Serialization/Scrubbers/ApplyScrubbers.cs index 4f94b49c57..ccd64e58fa 100644 --- a/src/Verify/Serialization/Scrubbers/ApplyScrubbers.cs +++ b/src/Verify/Serialization/Scrubbers/ApplyScrubbers.cs @@ -18,8 +18,11 @@ public static void ApplyForExtension(string extension, StringBuilder target, Ver } // Span scrubbers run first, then the legacy pass over the intermediate result. - // Path replacements and newline normalization stay after the legacy pass so - // legacy scrubbers keep seeing raw paths and may inject '\r'. + // The engine normalizes newlines up front, so a legacy scrubber sees text + // that already uses '\n' and one matching a literal "\r\n" will not fire. + // Newlines are normalized again at the end because a legacy scrubber can + // inject '\r'. Path replacements are the only part held back until after the + // legacy pass, so those scrubbers still see raw paths. var intermediate = ScrubEngine.Run(source, set, counter, settings.Context, applyDirectoryReplacements: false); if (!ReferenceEquals(intermediate, source)) { diff --git a/src/Verify/Serialization/Scrubbers/EngineScrubberSet.cs b/src/Verify/Serialization/Scrubbers/EngineScrubberSet.cs index 4793709421..041de7b4d5 100644 --- a/src/Verify/Serialization/Scrubbers/EngineScrubberSet.cs +++ b/src/Verify/Serialization/Scrubbers/EngineScrubberSet.cs @@ -17,11 +17,6 @@ sealed class EngineScrubberSet public bool HasLinePhase => LineDrops.Length > 0 || LineTransforms.Length > 0; - public bool IsEmpty => - LineDrops.Length == 0 && - LineTransforms.Length == 0 && - Inline.Length == 0; - static EngineScrubberSet empty = new([], [], []); // A set with no scrubbers, for a pass that only needs the engine's pinned diff --git a/src/Verify/Serialization/Scrubbers/ScrubEngine_Lines.cs b/src/Verify/Serialization/Scrubbers/ScrubEngine_Lines.cs index 1c4ca1eeab..90fdcaee16 100644 --- a/src/Verify/Serialization/Scrubbers/ScrubEngine_Lines.cs +++ b/src/Verify/Serialization/Scrubbers/ScrubEngine_Lines.cs @@ -16,8 +16,6 @@ readonly struct LineItem(int start, int end, string? fresh) public readonly int Start = start; public readonly int End = end; public readonly string? Fresh = fresh; - - public int Length => Fresh?.Length ?? End - Start; } // Returns the chunks representing the document after line drops and transforms, diff --git a/src/Verify/Serialization/Scrubbers/Scrubber.cs b/src/Verify/Serialization/Scrubbers/Scrubber.cs index c316416e62..9705c87e6c 100644 --- a/src/Verify/Serialization/Scrubbers/Scrubber.cs +++ b/src/Verify/Serialization/Scrubbers/Scrubber.cs @@ -100,11 +100,6 @@ or ScrubberKind.LineDropString Kind is ScrubberKind.LineTransformSpan or ScrubberKind.LineTransformString; - internal bool IsInline => - Kind is ScrubberKind.Replace - or ScrubberKind.Window - or ScrubberKind.Match; - /// /// Replace every occurrence of with . /// must be or . From c4ac93cc9ff6f6fb2fabd03fb4216acd8f9fb68d Mon Sep 17 00:00:00 2001 From: Simon Cropp Date: Sun, 19 Jul 2026 20:20:21 +1000 Subject: [PATCH 29/33] Delete release-notes-32.0.0.md --- release-notes-32.0.0.md | 125 ---------------------------------------- 1 file changed, 125 deletions(-) delete mode 100644 release-notes-32.0.0.md diff --git a/release-notes-32.0.0.md b/release-notes-32.0.0.md deleted file mode 100644 index 3c57035941..0000000000 --- a/release-notes-32.0.0.md +++ /dev/null @@ -1,125 +0,0 @@ -# Verify 32.0.0 release notes - -## Summary - -This release replaces the internal text-scrubbing pipeline with a new **span based scrub engine** and -adds a public **`Scrubber`** API. The engine materializes each document once, tracks it as chunks, and -quarantines replacements so scrubbers no longer re-process each other's output. The result is large CPU -and allocation reductions for the built-in scrubbers, plus a first-class way to define custom scrubbers. - -It is a **major version bump** because, although the public API is source compatible (see below), the -change is **behaviorally breaking in some configurations**: scrubber execution order and interaction are -now engine determined, so a subset of setups will produce different `.verified.` output and require -re-accepting snapshots. - -The full behavior is documented in [scrubbers.md](/docs/scrubbers.md#the-scrub-engine). - -## Why - -The old pipeline was `StringBuilder` based. Every registered scrubber re-round-tripped the whole -document (`ToString()` + rebuild, or chunk-walking with cross-chunk carryover buffers), so cost scaled -with `scrubbers x document size` and allocated heavily even when nothing matched. The new engine does a -single pass with vectorized candidate scanning and returns the original string unchanged (zero copy) -when no scrubber fired. - -## Performance - -Measured on a 31 KB document (p99 of a scan of 33,707 real `.verified.` files), AMD Ryzen 9 5900X, -.NET 10, `MemoryDiagnoser`. "Old" runs the pre-engine implementations over identical input. - -| Scenario | Old | New | Speedup | Allocation | -|---|--:|--:|--:|--:| -| Inline guid scan, no match | 105.5 us / 61.8 KB | 6.6 us / 1.2 KB | 16x | 51x less | -| Inline DateTime (ISO) scan, no match | 709.9 us / 122.4 KB | 72.7 us / 1.2 KB | 9.8x | 100x less | -| Inline DateTimeOffset, typical | 69.7 us / 15.5 KB | 4.1 us / 10.8 KB | 17x | — | -| `ScrubLinesContaining`, 508 lines | 16.2 us / 186 KB | 10.3 us / 59 KB | 1.6x | 3.2x less | -| Whole pipeline (guids + dates + line drops, one pass) | 884.0 us / 359 KB | 145.1 us / 103 KB | 6.1x | 3.5x less | -| Serialization string probe (per value, all misses) | 144 ns | 42 ns | 3.4x | — | - -The one scenario that regresses is predicate line removal (`ScrubLines(Func)`) on small and -medium documents (~0.5 us at p50, ~10 us at 1,000 lines) — the cost of the composable pipeline. It wins -at scale (5x at 10,000 lines) and is avoidable via the new span predicate overload or -`ScrubLinesContaining`. - -## Breaking changes - -### API (source compatible, with obsoletions) - -No public API was removed and no signatures changed incompatibly. However, the `ScrubberLocation` -parameter is now meaningless for the built-in scrub methods (ordering is engine determined), so those -overloads are marked `[Obsolete]`: - -`ScrubLinesContaining`, `ScrubLines`, `ScrubLinesWithReplace`, `ScrubEmptyLines`, `ScrubInlineGuids`, -`ScrubInlineDates`, `ScrubInlineDateTimes`, `ScrubInlineDateTimeOffsets`, `ScrubMachineName`, -`ScrubUserName` (at global, instance, extension mapped, and fluent levels). - -Existing calls still compile, but a call that passes a `ScrubberLocation` produces an obsolete warning. -Projects using `TreatWarningsAsErrors` will need to drop the `ScrubberLocation` argument. The -`ScrubberLocation` on the low-level `AddScrubber(Action, ...)` overloads is **unchanged** -and still honored. - -### Behavior (snapshot output) - -Even source-identical code can now produce different verified output in these cases. Where it applies, -re-accept the affected `.verified.` files. - -1. **Order among built-in scrubbers is engine determined.** `ScrubberLocation.First` / `.Last` on a - built-in scrub method is ignored. Registration level (global vs instance) no longer grants broad - priority; inline scrubbers order by match length. -2. **Built-in scrubbers run before legacy `AddScrubber(Action)` scrubbers**, regardless - of registration order. Custom `AddScrubber` delegates run after the engine and still see (and can - modify) its output. -3. **Multiple `ScrubLinesWithReplace` now compose in registration order (FIFO).** Previously the default - `ScrubberLocation.First` composed them in reverse. -4. **Quarantine.** Text produced by one built-in/`Scrubber` replacement is no longer re-examined by - another. Legacy `AddScrubber` delegates are unaffected and can still re-scrub. -5. **Word boundary rule unified to `!char.IsLetterOrDigit`.** Inline guid scrubbing previously used - `char.IsLetter || char.IsNumber`; a guid adjacent to an exotic numeric character (for example a - superscript digit) is now scrubbed where it previously was not. -6. **Text is newline-normalized (`\r\n` and lone `\r` become `\n`) before scrubbers run.** A legacy - `AddScrubber` delegate that matches a literal `"\r\n"` will no longer match. -7. **Directory/path replacement greedy trailing-separator absorption does not cross a quarantined - replacement boundary.** Only relevant when another scrubber's replacement sits immediately after a - scrubbed path. - -Scrubbers that cannot be expressed on the engine (positional buffer edits, full-document reformatters, -multi-line regex) should stay on the `AddScrubber(Action)` overloads, which are unchanged. - -## Migration - -1. Update to `Verify 32.0.0`. -2. If the build fails with obsolete warnings, remove the `ScrubberLocation` argument from built-in scrub - calls, for example `ScrubMachineName(ScrubberLocation.Last)` becomes `ScrubMachineName()`. -3. Run the test suite and re-accept any `.verified.` files that changed. Expected only in the - configurations listed under "Behavior" above; a suite that does not rely on scrubber ordering or - sugar-vs-legacy interaction should see no changes. -4. Optional: for hot custom line predicates, switch to the new span overloads to avoid a per-line string - allocation, for example `ScrubLines((ReadOnlySpan line) => ...)`. - -## New public API - -Create a `Scrubber` via static factories and register it with `AddScrubber(Scrubber)` at any level -(global, instance, extension mapped, fluent): - -```csharp -// Fixed-string replacement (optional ordinal comparison / word boundary) -settings.AddScrubber(Scrubber.Replace("find", "replacement")); - -// Sliding window matcher (used internally by the guid and date scrubbers) -settings.AddScrubber(Scrubber.Window(minLength: 26, maxLength: 26, (window, counter, context) => ...)); - -// General search matcher (locates the next match within a segment) -settings.AddScrubber(Scrubber.Match((segment, counter, context, out index, out length, out replacement) => ...)); - -// Line scoped -settings.AddScrubber(Scrubber.RemoveLinesContaining("token")); -settings.AddScrubber(Scrubber.RemoveEmptyLines()); -settings.AddScrubber(Scrubber.RemoveLines((ReadOnlySpan line) => ...)); // span predicate, no per-line alloc -settings.AddScrubber(Scrubber.ReplaceLines((ReadOnlySpan line) => ...)); // returns LineResult.Keep / Remove / Replace(text) -``` - -`ScrubLines` and `ScrubLinesWithReplace` also gained span-delegate overloads (`LineMatch` / -`LineReplace`); select them with an explicitly typed lambda parameter (`(ReadOnlySpan line) => ...`). Untyped -lambdas continue to bind the existing `string`-based overloads. - -See [scrubbers.md](/docs/scrubbers.md) for full semantics. From 81e01be4098674de8778b5e9c5e6605f3a128d50 Mon Sep 17 00:00:00 2001 From: Simon Cropp Date: Sun, 19 Jul 2026 21:10:51 +1000 Subject: [PATCH 30/33] . --- docs/guids.md | 2 +- docs/mdsource/scrubbers.source.md | 16 +++-- docs/members-throw.md | 8 +-- docs/named-tuples.md | 4 +- docs/obsolete-members.md | 6 +- docs/scrubbers.md | 72 +++++++++++-------- docs/serializer-settings.md | 44 ++++++------ ...ationTests.ScrubEngineMethods.verified.txt | 1 + .../Serialization/SerializationTests.cs | 69 ++++++++++++++---- ...Settings_ExtensionMappedGlobalScrubbers.cs | 30 ++++++++ .../VerifierSettings_GlobalScrubbers.cs | 30 ++++++++ ...ttings_ExtensionMappedInstanceScrubbers.cs | 30 ++++++++ .../VerifySettings_InstanceScrubbers.cs | 30 ++++++++ src/Verify/SettingsTask_Scrubbing.cs | 64 +++++++++++++++++ 14 files changed, 322 insertions(+), 84 deletions(-) create mode 100644 src/Verify.Tests/Serialization/SerializationTests.ScrubEngineMethods.verified.txt diff --git a/docs/guids.md b/docs/guids.md index db4a84c587..47d571705c 100644 --- a/docs/guids.md +++ b/docs/guids.md @@ -23,7 +23,7 @@ var target = new GuidTarget await Verify(target); ``` -snippet source | anchor +snippet source | anchor Results in the following: diff --git a/docs/mdsource/scrubbers.source.md b/docs/mdsource/scrubbers.source.md index 53a8aeb16f..e59a2ea335 100644 --- a/docs/mdsource/scrubbers.source.md +++ b/docs/mdsource/scrubbers.source.md @@ -12,12 +12,16 @@ Scrubbing is performed by two mechanisms: ## The scrub engine -A `Scrubber` is created via static factory methods and registered via `AddScrubber` at any [level](#Scrubber-levels): +Each engine operation is available as a `Scrub*` method at any [level](#Scrubber-levels): - * `Scrubber.Replace(find, replacement)`: replace every occurrence of a string. Supports an ordinal `StringComparison` (`Ordinal` or `OrdinalIgnoreCase`) and an optional word boundary requirement. A multi-pair overload replaces the longest matching find at any position. - * `Scrubber.Window(minLength, maxLength, matcher)`: slide a window over the text; the matcher returns a replacement or null. Used by the inline guid and date scrubbers. - * `Scrubber.Match(matcher, minLength, maxLength)`: custom search logic. The matcher locates the next match within a segment. - * `Scrubber.RemoveLinesContaining(...)`, `Scrubber.RemoveLines(...)`, `Scrubber.ReplaceLines(...)`, `Scrubber.RemoveEmptyLines()`: line scoped scrubbers. + * `ScrubReplace(find, replacement)`: replace every occurrence of a string. Supports an ordinal `StringComparison` (`Ordinal` or `OrdinalIgnoreCase`) and an optional word boundary requirement. A multi-pair overload replaces the longest matching find at any position. + * `ScrubWindow(minLength, maxLength, matcher)`: slide a window over the text; the matcher returns a replacement or null. Used by the inline guid and date scrubbers. + * `ScrubMatch(matcher, minLength, maxLength)`: custom search logic. The matcher locates the next match within a segment. + * `ScrubLinesContaining(...)`, `ScrubLines(...)`, `ScrubLinesWithReplace(...)`, `ScrubEmptyLines()`: line scoped scrubbing. + +snippet: ScrubEngine + +To reuse a scrubbing operation across levels or expose one from a helper library, create a `Scrubber` via the equivalent static factory methods (`Scrubber.Replace`, `Scrubber.Window`, `Scrubber.Match`, `Scrubber.RemoveLinesContaining`, `Scrubber.RemoveLines`, `Scrubber.ReplaceLines`, `Scrubber.RemoveEmptyLines`) and register it via `AddScrubber`: snippet: AddScrubberEngine @@ -185,7 +189,7 @@ snippet: Verify.XunitV3.Tests/Scrubbers/ScrubbersSample.Lines.verified.txt Scrubbers can be scoped to verified files with a matching extension by passing the extension as the first argument. The extension is specified without a leading dot: -snippet: AddScrubberEngineExtension +snippet: ScrubEngineExtension A scrubber registered this way runs only for verified files with that extension, while a scrubber registered without an extension runs for all of them. Extension scoping is available at every [level](#Scrubber-levels), and for the legacy `AddScrubber(Action)` overloads. diff --git a/docs/members-throw.md b/docs/members-throw.md index 309e9645b7..af7a5e2462 100644 --- a/docs/members-throw.md +++ b/docs/members-throw.md @@ -35,7 +35,7 @@ public Task CustomExceptionPropFluent() .IgnoreMembersThatThrow(); } ``` -snippet source | anchor +snippet source | anchor Or globally: @@ -45,7 +45,7 @@ Or globally: ```cs VerifierSettings.IgnoreMembersThatThrow(); ``` -snippet source | anchor +snippet source | anchor Result: @@ -82,7 +82,7 @@ public Task ExceptionMessagePropFluent() .IgnoreMembersThatThrow(_ => _.Message == "Ignore"); } ``` -snippet source | anchor +snippet source | anchor Or globally: @@ -92,7 +92,7 @@ Or globally: ```cs VerifierSettings.IgnoreMembersThatThrow(_ => _.Message == "Ignore"); ``` -snippet source | anchor +snippet source | anchor Result: diff --git a/docs/named-tuples.md b/docs/named-tuples.md index 0ddc8783b3..771438905e 100644 --- a/docs/named-tuples.md +++ b/docs/named-tuples.md @@ -19,7 +19,7 @@ Given a method that returns a named tuple: static (bool Member1, string Member2, string Member3) MethodWithNamedTuple() => (true, "A", "B"); ``` -snippet source | anchor +snippet source | anchor Can be verified: @@ -29,7 +29,7 @@ Can be verified: ```cs await VerifyTuple(() => MethodWithNamedTuple()); ``` -snippet source | anchor +snippet source | anchor Resulting in: diff --git a/docs/obsolete-members.md b/docs/obsolete-members.md index 506a190606..ce3bf78e2c 100644 --- a/docs/obsolete-members.md +++ b/docs/obsolete-members.md @@ -31,7 +31,7 @@ public Task WithObsoleteProp() return Verify(target); } ``` -snippet source | anchor +snippet source | anchor Result: @@ -79,7 +79,7 @@ public Task WithObsoletePropIncludedFluent() .IncludeObsoletes(); } ``` -snippet source | anchor +snippet source | anchor Or globally: @@ -89,7 +89,7 @@ Or globally: ```cs VerifierSettings.IncludeObsoletes(); ``` -snippet source | anchor +snippet source | anchor Result: diff --git a/docs/scrubbers.md b/docs/scrubbers.md index 8cf0b8cc15..1ae1d3ab75 100644 --- a/docs/scrubbers.md +++ b/docs/scrubbers.md @@ -19,33 +19,43 @@ Scrubbing is performed by two mechanisms: ## The scrub engine -A `Scrubber` is created via static factory methods and registered via `AddScrubber` at any [level](#Scrubber-levels): +Each engine operation is available as a `Scrub*` method at any [level](#Scrubber-levels): - * `Scrubber.Replace(find, replacement)`: replace every occurrence of a string. Supports an ordinal `StringComparison` (`Ordinal` or `OrdinalIgnoreCase`) and an optional word boundary requirement. A multi-pair overload replaces the longest matching find at any position. - * `Scrubber.Window(minLength, maxLength, matcher)`: slide a window over the text; the matcher returns a replacement or null. Used by the inline guid and date scrubbers. - * `Scrubber.Match(matcher, minLength, maxLength)`: custom search logic. The matcher locates the next match within a segment. - * `Scrubber.RemoveLinesContaining(...)`, `Scrubber.RemoveLines(...)`, `Scrubber.ReplaceLines(...)`, `Scrubber.RemoveEmptyLines()`: line scoped scrubbers. + * `ScrubReplace(find, replacement)`: replace every occurrence of a string. Supports an ordinal `StringComparison` (`Ordinal` or `OrdinalIgnoreCase`) and an optional word boundary requirement. A multi-pair overload replaces the longest matching find at any position. + * `ScrubWindow(minLength, maxLength, matcher)`: slide a window over the text; the matcher returns a replacement or null. Used by the inline guid and date scrubbers. + * `ScrubMatch(matcher, minLength, maxLength)`: custom search logic. The matcher locates the next match within a segment. + * `ScrubLinesContaining(...)`, `ScrubLines(...)`, `ScrubLinesWithReplace(...)`, `ScrubEmptyLines()`: line scoped scrubbing. - - + + ```cs -verifySettings.AddScrubber(Scrubber.Replace("abc", "xyz")); +verifySettings.ScrubReplace("abc", "xyz"); -verifySettings.AddScrubber( - Scrubber.Window( - minLength: 3, - maxLength: 10, - matcher: (window, _, _) => +verifySettings.ScrubWindow( + minLength: 3, + maxLength: 10, + matcher: (window, _, _) => + { + if (window.StartsWith("id-")) { - if (window.StartsWith("id-")) - { - return "{Id}"; - } + return "{Id}"; + } - return null; - })); + return null; + }); +``` +snippet source | anchor + + +To reuse a scrubbing operation across levels or expose one from a helper library, create a `Scrubber` via the equivalent static factory methods (`Scrubber.Replace`, `Scrubber.Window`, `Scrubber.Match`, `Scrubber.RemoveLinesContaining`, `Scrubber.RemoveLines`, `Scrubber.ReplaceLines`, `Scrubber.RemoveEmptyLines`) and register it via `AddScrubber`: + + + +```cs +var scrubber = Scrubber.Replace("abc", "xyz"); +verifySettings.AddScrubber(scrubber); ``` -snippet source | anchor +snippet source | anchor `ScrubLines` and `ScrubLinesWithReplace` (and the corresponding `Scrubber` factories) also accept span based delegates (`LineMatch` / `LineReplace`) that avoid allocating a string per line. Use an explicitly typed lambda parameter to select them, e.g. `ScrubLines((ReadOnlySpan line) => ...)`; untyped lambdas bind the string overloads. @@ -70,7 +80,7 @@ Instead of being executed by the engine, `AddScrubber(Action)` mu ```cs verifySettings.AddScrubber(_ => _.Remove(0, 100)); ``` -snippet source | anchor +snippet source | anchor Since these run after every engine scrubber, they can also modify text that the engine has already replaced. @@ -119,7 +129,7 @@ For example remove lines containing `text`: ```cs verifySettings.ScrubLines(line => line.Contains("text")); ``` -snippet source | anchor +snippet source | anchor @@ -134,7 +144,7 @@ For example remove lines containing `text1` or `text2` ```cs verifySettings.ScrubLinesContaining("text1", "text2"); ``` -snippet source | anchor +snippet source | anchor Case insensitive by default (`StringComparison.OrdinalIgnoreCase`). @@ -146,7 +156,7 @@ Case insensitive by default (`StringComparison.OrdinalIgnoreCase`). ```cs verifySettings.ScrubLinesContaining(StringComparison.Ordinal, "text1", "text2"); ``` -snippet source | anchor +snippet source | anchor @@ -161,7 +171,7 @@ For example converts lines to upper case: ```cs verifySettings.ScrubLinesWithReplace(line => line.ToUpper()); ``` -snippet source | anchor +snippet source | anchor @@ -174,7 +184,7 @@ Replaces `Environment.MachineName` with `TheMachineName`. ```cs verifySettings.ScrubMachineName(); ``` -snippet source | anchor +snippet source | anchor @@ -187,7 +197,7 @@ Replaces `Environment.UserName` with `TheUserName`. ```cs verifySettings.ScrubUserName(); ``` -snippet source | anchor +snippet source | anchor @@ -784,12 +794,12 @@ LineI Scrubbers can be scoped to verified files with a matching extension by passing the extension as the first argument. The extension is specified without a leading dot: - - + + ```cs -verifySettings.AddScrubber("json", Scrubber.Replace("abc", "xyz")); +verifySettings.ScrubReplace("json", "abc", "xyz"); ``` -snippet source | anchor +snippet source | anchor A scrubber registered this way runs only for verified files with that extension, while a scrubber registered without an extension runs for all of them. Extension scoping is available at every [level](#Scrubber-levels), and for the legacy `AddScrubber(Action)` overloads. diff --git a/docs/serializer-settings.md b/docs/serializer-settings.md index e3ea60a720..4eea2fda5a 100644 --- a/docs/serializer-settings.md +++ b/docs/serializer-settings.md @@ -495,7 +495,7 @@ public Task ScopedSerializerFluent() .AddExtraSettings(_ => _.TypeNameHandling = TypeNameHandling.All); } ``` -snippet source | anchor +snippet source | anchor Result: @@ -623,7 +623,7 @@ public Task IgnoreTypeFluent() .IgnoreMembersWithType(); } ``` -snippet source | anchor +snippet source | anchor Or globally: @@ -633,7 +633,7 @@ Or globally: ```cs VerifierSettings.IgnoreMembersWithType(); ``` -snippet source | anchor +snippet source | anchor Result: @@ -770,7 +770,7 @@ public Task ScrubTypeFluent() .ScrubMembersWithType(); } ``` -snippet source | anchor +snippet source | anchor Or globally: @@ -780,7 +780,7 @@ Or globally: ```cs VerifierSettings.ScrubMembersWithType(); ``` -snippet source | anchor +snippet source | anchor Result: @@ -859,7 +859,7 @@ public Task AddIgnoreInstanceFluent() .IgnoreInstance(_ => _.Property == "Ignore"); } ``` -snippet source | anchor +snippet source | anchor Or globally: @@ -869,7 +869,7 @@ Or globally: ```cs VerifierSettings.IgnoreInstance(_ => _.Property == "Ignore"); ``` -snippet source | anchor +snippet source | anchor Result: @@ -931,7 +931,7 @@ public Task AddScrubInstanceFluent() .ScrubInstance(_ => _.Property == "Ignore"); } ``` -snippet source | anchor +snippet source | anchor Or globally: @@ -941,7 +941,7 @@ Or globally: ```cs VerifierSettings.ScrubInstance(_ => _.Property == "Ignore"); ``` -snippet source | anchor +snippet source | anchor Result: @@ -1004,7 +1004,7 @@ public Task IgnoreMemberByExpressionFluent() _ => _.PropertyThatThrows); } ``` -snippet source | anchor +snippet source | anchor Or globally @@ -1019,7 +1019,7 @@ VerifierSettings.IgnoreMembers( _ => _.GetOnlyProperty, _ => _.PropertyThatThrows); ``` -snippet source | anchor +snippet source | anchor Result: @@ -1079,7 +1079,7 @@ public Task ScrubMemberByExpressionFluent() _ => _.PropertyThatThrows); } ``` -snippet source | anchor +snippet source | anchor Or globally @@ -1094,7 +1094,7 @@ VerifierSettings.ScrubMembers( _ => _.GetOnlyProperty, _ => _.PropertyThatThrows); ``` -snippet source | anchor +snippet source | anchor Result: @@ -1173,7 +1173,7 @@ public Task IgnoreMemberByNameFluent() .IgnoreMember(_ => _.PropertyThatThrows); } ``` -snippet source | anchor +snippet source | anchor Or globally: @@ -1193,7 +1193,7 @@ VerifierSettings.IgnoreMember("Field"); // For a specific type with expression VerifierSettings.IgnoreMember(_ => _.PropertyThatThrows); ``` -snippet source | anchor +snippet source | anchor Result: @@ -1268,7 +1268,7 @@ public Task ScrubMemberByNameFluent() .ScrubMember(_ => _.PropertyThatThrows); } ``` -snippet source | anchor +snippet source | anchor Or globally: @@ -1288,7 +1288,7 @@ VerifierSettings.ScrubMember("Field"); // For a specific type with expression VerifierSettings.ScrubMember(_ => _.PropertyThatThrows); ``` -snippet source | anchor +snippet source | anchor Result: @@ -1381,7 +1381,7 @@ public Task IgnoreDictionaryByPredicate() return Verify(target, settings); } ``` -snippet source | anchor +snippet source | anchor Or globally: @@ -1393,7 +1393,7 @@ VerifierSettings.IgnoreMembers( _=>_.DeclaringType == typeof(TargetClass) && _.Name == "Proprty"); ``` -snippet source | anchor +snippet source | anchor Result: @@ -1482,7 +1482,7 @@ public Task ScrubDictionaryByPredicate() return Verify(target, settings); } ``` -snippet source | anchor +snippet source | anchor Or globally: @@ -1494,7 +1494,7 @@ VerifierSettings.ScrubMembers( _=>_.DeclaringType == typeof(TargetClass) && _.Name == "Proprty"); ``` -snippet source | anchor +snippet source | anchor Result: @@ -1548,7 +1548,7 @@ public Task MemberConverterByExpression() return Verify(input); } ``` -snippet source | anchor +snippet source | anchor diff --git a/src/Verify.Tests/Serialization/SerializationTests.ScrubEngineMethods.verified.txt b/src/Verify.Tests/Serialization/SerializationTests.ScrubEngineMethods.verified.txt new file mode 100644 index 0000000000..e03f786bd5 --- /dev/null +++ b/src/Verify.Tests/Serialization/SerializationTests.ScrubEngineMethods.verified.txt @@ -0,0 +1 @@ +{Value} {Id} xyz \ No newline at end of file diff --git a/src/Verify.Tests/Serialization/SerializationTests.cs b/src/Verify.Tests/Serialization/SerializationTests.cs index d52b8d89e0..f8be22a266 100644 --- a/src/Verify.Tests/Serialization/SerializationTests.cs +++ b/src/Verify.Tests/Serialization/SerializationTests.cs @@ -2023,6 +2023,39 @@ public Task NewLineNotEscapedInProperty() => Property = "a\r\nb\\nc" }); + [Fact] + public Task ScrubEngineMethods() => + Verify("value id-123 abc") + .ScrubReplace("abc", "xyz") + .ScrubWindow( + minLength: 4, + maxLength: 6, + matcher: (window, _, _) => + { + if (window.Length == 6 && + window.StartsWith("id-")) + { + return "{Id}"; + } + + return null; + }) + .ScrubMatch( + (ReadOnlySpan segment, Counter _, IReadOnlyDictionary _, out int index, out int length, out string? replacement) => + { + index = segment.IndexOf("value".AsSpan()); + if (index == -1) + { + length = 0; + replacement = null; + return false; + } + + length = "value".Length; + replacement = "{Value}"; + return true; + }); + // ReSharper disable once UnusedMember.Local static void List() { @@ -2070,29 +2103,35 @@ static void List() #endregion - #region AddScrubberEngine + #region ScrubEngine - verifySettings.AddScrubber(Scrubber.Replace("abc", "xyz")); + verifySettings.ScrubReplace("abc", "xyz"); - verifySettings.AddScrubber( - Scrubber.Window( - minLength: 3, - maxLength: 10, - matcher: (window, _, _) => + verifySettings.ScrubWindow( + minLength: 3, + maxLength: 10, + matcher: (window, _, _) => + { + if (window.StartsWith("id-")) { - if (window.StartsWith("id-")) - { - return "{Id}"; - } + return "{Id}"; + } - return null; - })); + return null; + }); + + #endregion + + #region AddScrubberEngine + + var scrubber = Scrubber.Replace("abc", "xyz"); + verifySettings.AddScrubber(scrubber); #endregion - #region AddScrubberEngineExtension + #region ScrubEngineExtension - verifySettings.AddScrubber("json", Scrubber.Replace("abc", "xyz")); + verifySettings.ScrubReplace("json", "abc", "xyz"); #endregion } diff --git a/src/Verify/Serialization/Scrubbers/VerifierSettings_ExtensionMappedGlobalScrubbers.cs b/src/Verify/Serialization/Scrubbers/VerifierSettings_ExtensionMappedGlobalScrubbers.cs index f16c5aa342..e31efcbe34 100644 --- a/src/Verify/Serialization/Scrubbers/VerifierSettings_ExtensionMappedGlobalScrubbers.cs +++ b/src/Verify/Serialization/Scrubbers/VerifierSettings_ExtensionMappedGlobalScrubbers.cs @@ -122,4 +122,34 @@ public static void ScrubMachineName(string extension) => /// public static void ScrubUserName(string extension) => AddScrubber(extension, UserMachineScrubber.UserScrubber()); + + /// + /// Replace every occurrence of with . + /// must be or . + /// + public static void ScrubReplace(string extension, string find, string replacement, StringComparison comparison = StringComparison.Ordinal, bool requireWordBoundary = false) => + AddScrubber(extension, Scrubber.Replace(find, replacement, comparison, requireWordBoundary)); + + /// + /// Replace every occurrence of each Find with its Replacement. + /// At a given position the longest matching Find wins. + /// must be or . + /// + public static void ScrubReplace(string extension, StringComparison comparison, bool requireWordBoundary, params (string Find, string Replacement)[] pairs) => + AddScrubber(extension, Scrubber.Replace(comparison, requireWordBoundary, pairs)); + + /// + /// Match candidate windows of text between and characters. + /// At each position the engine tries the longest window first. + /// + public static void ScrubWindow(string extension, int minLength, int maxLength, WindowMatch matcher, bool requireWordBoundary = false) => + AddScrubber(extension, Scrubber.Window(minLength, maxLength, matcher, requireWordBoundary)); + + /// + /// Find matches using custom search logic. + /// : segments shorter than this are skipped (null scans everything). + /// : used for ordering only; null (unknown) runs before all known length scrubbers. + /// + public static void ScrubMatch(string extension, SegmentMatch matcher, int? minLength = null, int? maxLength = null) => + AddScrubber(extension, Scrubber.Match(matcher, minLength, maxLength)); } diff --git a/src/Verify/Serialization/Scrubbers/VerifierSettings_GlobalScrubbers.cs b/src/Verify/Serialization/Scrubbers/VerifierSettings_GlobalScrubbers.cs index 07441bf67e..a6dd7ced8f 100644 --- a/src/Verify/Serialization/Scrubbers/VerifierSettings_GlobalScrubbers.cs +++ b/src/Verify/Serialization/Scrubbers/VerifierSettings_GlobalScrubbers.cs @@ -165,4 +165,34 @@ public static void ScrubMachineName() => /// public static void ScrubUserName() => AddScrubber(UserMachineScrubber.UserScrubber()); + + /// + /// Replace every occurrence of with . + /// must be or . + /// + public static void ScrubReplace(string find, string replacement, StringComparison comparison = StringComparison.Ordinal, bool requireWordBoundary = false) => + AddScrubber(Scrubber.Replace(find, replacement, comparison, requireWordBoundary)); + + /// + /// Replace every occurrence of each Find with its Replacement. + /// At a given position the longest matching Find wins. + /// must be or . + /// + public static void ScrubReplace(StringComparison comparison, bool requireWordBoundary, params (string Find, string Replacement)[] pairs) => + AddScrubber(Scrubber.Replace(comparison, requireWordBoundary, pairs)); + + /// + /// Match candidate windows of text between and characters. + /// At each position the engine tries the longest window first. + /// + public static void ScrubWindow(int minLength, int maxLength, WindowMatch matcher, bool requireWordBoundary = false) => + AddScrubber(Scrubber.Window(minLength, maxLength, matcher, requireWordBoundary)); + + /// + /// Find matches using custom search logic. + /// : segments shorter than this are skipped (null scans everything). + /// : used for ordering only; null (unknown) runs before all known length scrubbers. + /// + public static void ScrubMatch(SegmentMatch matcher, int? minLength = null, int? maxLength = null) => + AddScrubber(Scrubber.Match(matcher, minLength, maxLength)); } diff --git a/src/Verify/Serialization/Scrubbers/VerifySettings_ExtensionMappedInstanceScrubbers.cs b/src/Verify/Serialization/Scrubbers/VerifySettings_ExtensionMappedInstanceScrubbers.cs index 2e4ae83660..a1d0ef8105 100644 --- a/src/Verify/Serialization/Scrubbers/VerifySettings_ExtensionMappedInstanceScrubbers.cs +++ b/src/Verify/Serialization/Scrubbers/VerifySettings_ExtensionMappedInstanceScrubbers.cs @@ -120,4 +120,34 @@ public void ScrubLinesWithReplace(string extension, LineReplace replaceLine) => /// public void ScrubEmptyLines(string extension) => AddScrubber(extension, Scrubber.RemoveEmptyLines()); + + /// + /// Replace every occurrence of with . + /// must be or . + /// + public void ScrubReplace(string extension, string find, string replacement, StringComparison comparison = StringComparison.Ordinal, bool requireWordBoundary = false) => + AddScrubber(extension, Scrubber.Replace(find, replacement, comparison, requireWordBoundary)); + + /// + /// Replace every occurrence of each Find with its Replacement. + /// At a given position the longest matching Find wins. + /// must be or . + /// + public void ScrubReplace(string extension, StringComparison comparison, bool requireWordBoundary, params (string Find, string Replacement)[] pairs) => + AddScrubber(extension, Scrubber.Replace(comparison, requireWordBoundary, pairs)); + + /// + /// Match candidate windows of text between and characters. + /// At each position the engine tries the longest window first. + /// + public void ScrubWindow(string extension, int minLength, int maxLength, WindowMatch matcher, bool requireWordBoundary = false) => + AddScrubber(extension, Scrubber.Window(minLength, maxLength, matcher, requireWordBoundary)); + + /// + /// Find matches using custom search logic. + /// : segments shorter than this are skipped (null scans everything). + /// : used for ordering only; null (unknown) runs before all known length scrubbers. + /// + public void ScrubMatch(string extension, SegmentMatch matcher, int? minLength = null, int? maxLength = null) => + AddScrubber(extension, Scrubber.Match(matcher, minLength, maxLength)); } diff --git a/src/Verify/Serialization/Scrubbers/VerifySettings_InstanceScrubbers.cs b/src/Verify/Serialization/Scrubbers/VerifySettings_InstanceScrubbers.cs index 68c4550de8..de73a6cdf7 100644 --- a/src/Verify/Serialization/Scrubbers/VerifySettings_InstanceScrubbers.cs +++ b/src/Verify/Serialization/Scrubbers/VerifySettings_InstanceScrubbers.cs @@ -200,4 +200,34 @@ public void ScrubEmptyLines() => /// public void ScrubLinesContaining(params string[] stringToMatch) => ScrubLinesContaining(StringComparison.OrdinalIgnoreCase, stringToMatch); + + /// + /// Replace every occurrence of with . + /// must be or . + /// + public void ScrubReplace(string find, string replacement, StringComparison comparison = StringComparison.Ordinal, bool requireWordBoundary = false) => + AddScrubber(Scrubber.Replace(find, replacement, comparison, requireWordBoundary)); + + /// + /// Replace every occurrence of each Find with its Replacement. + /// At a given position the longest matching Find wins. + /// must be or . + /// + public void ScrubReplace(StringComparison comparison, bool requireWordBoundary, params (string Find, string Replacement)[] pairs) => + AddScrubber(Scrubber.Replace(comparison, requireWordBoundary, pairs)); + + /// + /// Match candidate windows of text between and characters. + /// At each position the engine tries the longest window first. + /// + public void ScrubWindow(int minLength, int maxLength, WindowMatch matcher, bool requireWordBoundary = false) => + AddScrubber(Scrubber.Window(minLength, maxLength, matcher, requireWordBoundary)); + + /// + /// Find matches using custom search logic. + /// : segments shorter than this are skipped (null scans everything). + /// : used for ordering only; null (unknown) runs before all known length scrubbers. + /// + public void ScrubMatch(SegmentMatch matcher, int? minLength = null, int? maxLength = null) => + AddScrubber(Scrubber.Match(matcher, minLength, maxLength)); } diff --git a/src/Verify/SettingsTask_Scrubbing.cs b/src/Verify/SettingsTask_Scrubbing.cs index 2fc9ce4d8a..231df979dc 100644 --- a/src/Verify/SettingsTask_Scrubbing.cs +++ b/src/Verify/SettingsTask_Scrubbing.cs @@ -263,4 +263,68 @@ public SettingsTask ScrubLinesContaining(params string[] stringToMatch) CurrentSettings.ScrubLinesContaining(stringToMatch); return this; } + + /// + [Pure] + public SettingsTask ScrubReplace(string find, string replacement, StringComparison comparison = StringComparison.Ordinal, bool requireWordBoundary = false) + { + CurrentSettings.ScrubReplace(find, replacement, comparison, requireWordBoundary); + return this; + } + + /// + [Pure] + public SettingsTask ScrubReplace(string extension, string find, string replacement, StringComparison comparison = StringComparison.Ordinal, bool requireWordBoundary = false) + { + CurrentSettings.ScrubReplace(extension, find, replacement, comparison, requireWordBoundary); + return this; + } + + /// + [Pure] + public SettingsTask ScrubReplace(StringComparison comparison, bool requireWordBoundary, params (string Find, string Replacement)[] pairs) + { + CurrentSettings.ScrubReplace(comparison, requireWordBoundary, pairs); + return this; + } + + /// + [Pure] + public SettingsTask ScrubReplace(string extension, StringComparison comparison, bool requireWordBoundary, params (string Find, string Replacement)[] pairs) + { + CurrentSettings.ScrubReplace(extension, comparison, requireWordBoundary, pairs); + return this; + } + + /// + [Pure] + public SettingsTask ScrubWindow(int minLength, int maxLength, WindowMatch matcher, bool requireWordBoundary = false) + { + CurrentSettings.ScrubWindow(minLength, maxLength, matcher, requireWordBoundary); + return this; + } + + /// + [Pure] + public SettingsTask ScrubWindow(string extension, int minLength, int maxLength, WindowMatch matcher, bool requireWordBoundary = false) + { + CurrentSettings.ScrubWindow(extension, minLength, maxLength, matcher, requireWordBoundary); + return this; + } + + /// + [Pure] + public SettingsTask ScrubMatch(SegmentMatch matcher, int? minLength = null, int? maxLength = null) + { + CurrentSettings.ScrubMatch(matcher, minLength, maxLength); + return this; + } + + /// + [Pure] + public SettingsTask ScrubMatch(string extension, SegmentMatch matcher, int? minLength = null, int? maxLength = null) + { + CurrentSettings.ScrubMatch(extension, matcher, minLength, maxLength); + return this; + } } From bbfd1b5d1d9ea458eec4d75af2ef46aa39d8cb7b Mon Sep 17 00:00:00 2001 From: Simon Cropp Date: Sun, 19 Jul 2026 21:35:08 +1000 Subject: [PATCH 31/33] . --- docs/guids.md | 2 +- docs/mdsource/scrubbers.source.md | 8 +--- docs/members-throw.md | 8 ++-- docs/named-tuples.md | 4 +- docs/obsolete-members.md | 6 +-- docs/scrubbers.md | 17 ++----- docs/serializer-settings.md | 44 +++++++++---------- .../Serialization/SerializationTests.cs | 7 --- .../Serialization/Scrubbers/Scrubber.cs | 4 +- .../Scrubbers/Scrubber_Delegates.cs | 2 +- ...Settings_ExtensionMappedGlobalScrubbers.cs | 2 +- .../VerifierSettings_GlobalScrubbers.cs | 2 +- ...ttings_ExtensionMappedInstanceScrubbers.cs | 2 +- .../VerifySettings_InstanceScrubbers.cs | 2 +- src/Verify/SettingsTask_Scrubbing.cs | 4 +- 15 files changed, 45 insertions(+), 69 deletions(-) diff --git a/docs/guids.md b/docs/guids.md index 47d571705c..2a3d9e87ba 100644 --- a/docs/guids.md +++ b/docs/guids.md @@ -23,7 +23,7 @@ var target = new GuidTarget await Verify(target); ``` -snippet source | anchor +snippet source | anchor Results in the following: diff --git a/docs/mdsource/scrubbers.source.md b/docs/mdsource/scrubbers.source.md index e59a2ea335..a868bc904f 100644 --- a/docs/mdsource/scrubbers.source.md +++ b/docs/mdsource/scrubbers.source.md @@ -6,7 +6,7 @@ Multiple scrubbers [can be defined at multiple levels](#Scrubber-levels). Scrubbing is performed by two mechanisms: - * **The scrub engine**: a span based engine that executes `Scrubber` definitions. All built-in scrubbing (`ScrubLinesContaining`, `ScrubInlineGuids`, `ScrubMachineName`, etc) runs on the engine. + * **The scrub engine**: a span based engine that performs all built-in scrubbing (`ScrubLinesContaining`, `ScrubInlineGuids`, `ScrubMachineName`, etc). * **Legacy scrubbers**: `AddScrubber(Action)` overloads. These run after the engine, and only when at least one is registered. @@ -21,11 +21,7 @@ Each engine operation is available as a `Scrub*` method at any [level](#Scrubber snippet: ScrubEngine -To reuse a scrubbing operation across levels or expose one from a helper library, create a `Scrubber` via the equivalent static factory methods (`Scrubber.Replace`, `Scrubber.Window`, `Scrubber.Match`, `Scrubber.RemoveLinesContaining`, `Scrubber.RemoveLines`, `Scrubber.ReplaceLines`, `Scrubber.RemoveEmptyLines`) and register it via `AddScrubber`: - -snippet: AddScrubberEngine - -`ScrubLines` and `ScrubLinesWithReplace` (and the corresponding `Scrubber` factories) also accept span based delegates (`LineMatch` / `LineReplace`) that avoid allocating a string per line. Use an explicitly typed lambda parameter to select them, e.g. `ScrubLines((ReadOnlySpan line) => ...)`; untyped lambdas bind the string overloads. +`ScrubLines` and `ScrubLinesWithReplace` also accept span based delegates (`LineMatch` / `LineReplace`) that avoid allocating a string per line. Use an explicitly typed lambda parameter to select them, e.g. `ScrubLines((ReadOnlySpan line) => ...)`; untyped lambdas bind the string overloads. Engine semantics: diff --git a/docs/members-throw.md b/docs/members-throw.md index af7a5e2462..fea107e643 100644 --- a/docs/members-throw.md +++ b/docs/members-throw.md @@ -35,7 +35,7 @@ public Task CustomExceptionPropFluent() .IgnoreMembersThatThrow(); } ``` -snippet source | anchor +snippet source | anchor Or globally: @@ -45,7 +45,7 @@ Or globally: ```cs VerifierSettings.IgnoreMembersThatThrow(); ``` -snippet source | anchor +snippet source | anchor Result: @@ -82,7 +82,7 @@ public Task ExceptionMessagePropFluent() .IgnoreMembersThatThrow(_ => _.Message == "Ignore"); } ``` -snippet source | anchor +snippet source | anchor Or globally: @@ -92,7 +92,7 @@ Or globally: ```cs VerifierSettings.IgnoreMembersThatThrow(_ => _.Message == "Ignore"); ``` -snippet source | anchor +snippet source | anchor Result: diff --git a/docs/named-tuples.md b/docs/named-tuples.md index 771438905e..cbc2a40f2d 100644 --- a/docs/named-tuples.md +++ b/docs/named-tuples.md @@ -19,7 +19,7 @@ Given a method that returns a named tuple: static (bool Member1, string Member2, string Member3) MethodWithNamedTuple() => (true, "A", "B"); ``` -snippet source | anchor +snippet source | anchor Can be verified: @@ -29,7 +29,7 @@ Can be verified: ```cs await VerifyTuple(() => MethodWithNamedTuple()); ``` -snippet source | anchor +snippet source | anchor Resulting in: diff --git a/docs/obsolete-members.md b/docs/obsolete-members.md index ce3bf78e2c..9a8412f1a1 100644 --- a/docs/obsolete-members.md +++ b/docs/obsolete-members.md @@ -31,7 +31,7 @@ public Task WithObsoleteProp() return Verify(target); } ``` -snippet source | anchor +snippet source | anchor Result: @@ -79,7 +79,7 @@ public Task WithObsoletePropIncludedFluent() .IncludeObsoletes(); } ``` -snippet source | anchor +snippet source | anchor Or globally: @@ -89,7 +89,7 @@ Or globally: ```cs VerifierSettings.IncludeObsoletes(); ``` -snippet source | anchor +snippet source | anchor Result: diff --git a/docs/scrubbers.md b/docs/scrubbers.md index 1ae1d3ab75..ccbd5d869d 100644 --- a/docs/scrubbers.md +++ b/docs/scrubbers.md @@ -13,7 +13,7 @@ Multiple scrubbers [can be defined at multiple levels](#Scrubber-levels). Scrubbing is performed by two mechanisms: - * **The scrub engine**: a span based engine that executes `Scrubber` definitions. All built-in scrubbing (`ScrubLinesContaining`, `ScrubInlineGuids`, `ScrubMachineName`, etc) runs on the engine. + * **The scrub engine**: a span based engine that performs all built-in scrubbing (`ScrubLinesContaining`, `ScrubInlineGuids`, `ScrubMachineName`, etc). * **Legacy scrubbers**: `AddScrubber(Action)` overloads. These run after the engine, and only when at least one is registered. @@ -47,18 +47,7 @@ verifySettings.ScrubWindow( snippet source | anchor -To reuse a scrubbing operation across levels or expose one from a helper library, create a `Scrubber` via the equivalent static factory methods (`Scrubber.Replace`, `Scrubber.Window`, `Scrubber.Match`, `Scrubber.RemoveLinesContaining`, `Scrubber.RemoveLines`, `Scrubber.ReplaceLines`, `Scrubber.RemoveEmptyLines`) and register it via `AddScrubber`: - - - -```cs -var scrubber = Scrubber.Replace("abc", "xyz"); -verifySettings.AddScrubber(scrubber); -``` -snippet source | anchor - - -`ScrubLines` and `ScrubLinesWithReplace` (and the corresponding `Scrubber` factories) also accept span based delegates (`LineMatch` / `LineReplace`) that avoid allocating a string per line. Use an explicitly typed lambda parameter to select them, e.g. `ScrubLines((ReadOnlySpan line) => ...)`; untyped lambdas bind the string overloads. +`ScrubLines` and `ScrubLinesWithReplace` also accept span based delegates (`LineMatch` / `LineReplace`) that avoid allocating a string per line. Use an explicitly typed lambda parameter to select them, e.g. `ScrubLines((ReadOnlySpan line) => ...)`; untyped lambdas bind the string overloads. Engine semantics: @@ -799,7 +788,7 @@ Scrubbers can be scoped to verified files with a matching extension by passing t ```cs verifySettings.ScrubReplace("json", "abc", "xyz"); ``` -snippet source | anchor +snippet source | anchor A scrubber registered this way runs only for verified files with that extension, while a scrubber registered without an extension runs for all of them. Extension scoping is available at every [level](#Scrubber-levels), and for the legacy `AddScrubber(Action)` overloads. diff --git a/docs/serializer-settings.md b/docs/serializer-settings.md index 4eea2fda5a..f7cda568a8 100644 --- a/docs/serializer-settings.md +++ b/docs/serializer-settings.md @@ -495,7 +495,7 @@ public Task ScopedSerializerFluent() .AddExtraSettings(_ => _.TypeNameHandling = TypeNameHandling.All); } ``` -snippet source | anchor +snippet source | anchor Result: @@ -623,7 +623,7 @@ public Task IgnoreTypeFluent() .IgnoreMembersWithType(); } ``` -snippet source | anchor +snippet source | anchor Or globally: @@ -633,7 +633,7 @@ Or globally: ```cs VerifierSettings.IgnoreMembersWithType(); ``` -snippet source | anchor +snippet source | anchor Result: @@ -770,7 +770,7 @@ public Task ScrubTypeFluent() .ScrubMembersWithType(); } ``` -snippet source | anchor +snippet source | anchor Or globally: @@ -780,7 +780,7 @@ Or globally: ```cs VerifierSettings.ScrubMembersWithType(); ``` -snippet source | anchor +snippet source | anchor Result: @@ -859,7 +859,7 @@ public Task AddIgnoreInstanceFluent() .IgnoreInstance(_ => _.Property == "Ignore"); } ``` -snippet source | anchor +snippet source | anchor Or globally: @@ -869,7 +869,7 @@ Or globally: ```cs VerifierSettings.IgnoreInstance(_ => _.Property == "Ignore"); ``` -snippet source | anchor +snippet source | anchor Result: @@ -931,7 +931,7 @@ public Task AddScrubInstanceFluent() .ScrubInstance(_ => _.Property == "Ignore"); } ``` -snippet source | anchor +snippet source | anchor Or globally: @@ -941,7 +941,7 @@ Or globally: ```cs VerifierSettings.ScrubInstance(_ => _.Property == "Ignore"); ``` -snippet source | anchor +snippet source | anchor Result: @@ -1004,7 +1004,7 @@ public Task IgnoreMemberByExpressionFluent() _ => _.PropertyThatThrows); } ``` -snippet source | anchor +snippet source | anchor Or globally @@ -1019,7 +1019,7 @@ VerifierSettings.IgnoreMembers( _ => _.GetOnlyProperty, _ => _.PropertyThatThrows); ``` -snippet source | anchor +snippet source | anchor Result: @@ -1079,7 +1079,7 @@ public Task ScrubMemberByExpressionFluent() _ => _.PropertyThatThrows); } ``` -snippet source | anchor +snippet source | anchor Or globally @@ -1094,7 +1094,7 @@ VerifierSettings.ScrubMembers( _ => _.GetOnlyProperty, _ => _.PropertyThatThrows); ``` -snippet source | anchor +snippet source | anchor Result: @@ -1173,7 +1173,7 @@ public Task IgnoreMemberByNameFluent() .IgnoreMember(_ => _.PropertyThatThrows); } ``` -snippet source | anchor +snippet source | anchor Or globally: @@ -1193,7 +1193,7 @@ VerifierSettings.IgnoreMember("Field"); // For a specific type with expression VerifierSettings.IgnoreMember(_ => _.PropertyThatThrows); ``` -snippet source | anchor +snippet source | anchor Result: @@ -1268,7 +1268,7 @@ public Task ScrubMemberByNameFluent() .ScrubMember(_ => _.PropertyThatThrows); } ``` -snippet source | anchor +snippet source | anchor Or globally: @@ -1288,7 +1288,7 @@ VerifierSettings.ScrubMember("Field"); // For a specific type with expression VerifierSettings.ScrubMember(_ => _.PropertyThatThrows); ``` -snippet source | anchor +snippet source | anchor Result: @@ -1381,7 +1381,7 @@ public Task IgnoreDictionaryByPredicate() return Verify(target, settings); } ``` -snippet source | anchor +snippet source | anchor Or globally: @@ -1393,7 +1393,7 @@ VerifierSettings.IgnoreMembers( _=>_.DeclaringType == typeof(TargetClass) && _.Name == "Proprty"); ``` -snippet source | anchor +snippet source | anchor Result: @@ -1482,7 +1482,7 @@ public Task ScrubDictionaryByPredicate() return Verify(target, settings); } ``` -snippet source | anchor +snippet source | anchor Or globally: @@ -1494,7 +1494,7 @@ VerifierSettings.ScrubMembers( _=>_.DeclaringType == typeof(TargetClass) && _.Name == "Proprty"); ``` -snippet source | anchor +snippet source | anchor Result: @@ -1548,7 +1548,7 @@ public Task MemberConverterByExpression() return Verify(input); } ``` -snippet source | anchor +snippet source | anchor diff --git a/src/Verify.Tests/Serialization/SerializationTests.cs b/src/Verify.Tests/Serialization/SerializationTests.cs index f8be22a266..8a624fd50a 100644 --- a/src/Verify.Tests/Serialization/SerializationTests.cs +++ b/src/Verify.Tests/Serialization/SerializationTests.cs @@ -2122,13 +2122,6 @@ static void List() #endregion - #region AddScrubberEngine - - var scrubber = Scrubber.Replace("abc", "xyz"); - verifySettings.AddScrubber(scrubber); - - #endregion - #region ScrubEngineExtension verifySettings.ScrubReplace("json", "abc", "xyz"); diff --git a/src/Verify/Serialization/Scrubbers/Scrubber.cs b/src/Verify/Serialization/Scrubbers/Scrubber.cs index 9705c87e6c..84026d7db4 100644 --- a/src/Verify/Serialization/Scrubbers/Scrubber.cs +++ b/src/Verify/Serialization/Scrubbers/Scrubber.cs @@ -1,5 +1,3 @@ -namespace VerifyTests; - /// /// Defines a scrubbing operation executed by the span based scrub engine. /// @@ -19,7 +17,7 @@ namespace VerifyTests; /// Word boundary: when required, a match is rejected if the character on either side is a letter or digit. /// /// -public sealed class Scrubber +sealed class Scrubber { internal ScrubberKind Kind { get; } internal int MinLength { get; } diff --git a/src/Verify/Serialization/Scrubbers/Scrubber_Delegates.cs b/src/Verify/Serialization/Scrubbers/Scrubber_Delegates.cs index 4894b88cb9..b65ac4ecf0 100644 --- a/src/Verify/Serialization/Scrubbers/Scrubber_Delegates.cs +++ b/src/Verify/Serialization/Scrubbers/Scrubber_Delegates.cs @@ -2,7 +2,7 @@ namespace VerifyTests; /// /// Attempts to match a candidate window of text. -/// The window length is between the minLength and maxLength of the owning and never contains a line break. +/// The window length is between the minLength and maxLength given at registration and never contains a line break. /// Return the replacement text, or null when the window is not a match. /// public delegate string? WindowMatch( diff --git a/src/Verify/Serialization/Scrubbers/VerifierSettings_ExtensionMappedGlobalScrubbers.cs b/src/Verify/Serialization/Scrubbers/VerifierSettings_ExtensionMappedGlobalScrubbers.cs index e31efcbe34..2c17decd9e 100644 --- a/src/Verify/Serialization/Scrubbers/VerifierSettings_ExtensionMappedGlobalScrubbers.cs +++ b/src/Verify/Serialization/Scrubbers/VerifierSettings_ExtensionMappedGlobalScrubbers.cs @@ -9,7 +9,7 @@ public static partial class VerifierSettings /// /// Add a that applies to all verified files with a matching extension. /// - public static void AddScrubber(string extension, Scrubber scrubber) + internal static void AddScrubber(string extension, Scrubber scrubber) { InnerVerifier.ThrowIfVerifyHasBeenRun(); Ensure.NotNull(scrubber); diff --git a/src/Verify/Serialization/Scrubbers/VerifierSettings_GlobalScrubbers.cs b/src/Verify/Serialization/Scrubbers/VerifierSettings_GlobalScrubbers.cs index a6dd7ced8f..2256554fa6 100644 --- a/src/Verify/Serialization/Scrubbers/VerifierSettings_GlobalScrubbers.cs +++ b/src/Verify/Serialization/Scrubbers/VerifierSettings_GlobalScrubbers.cs @@ -9,7 +9,7 @@ public static partial class VerifierSettings /// /// Add a that applies to all verified files. /// - public static void AddScrubber(Scrubber scrubber) + internal static void AddScrubber(Scrubber scrubber) { InnerVerifier.ThrowIfVerifyHasBeenRun(); Ensure.NotNull(scrubber); diff --git a/src/Verify/Serialization/Scrubbers/VerifySettings_ExtensionMappedInstanceScrubbers.cs b/src/Verify/Serialization/Scrubbers/VerifySettings_ExtensionMappedInstanceScrubbers.cs index a1d0ef8105..db9fdb6f94 100644 --- a/src/Verify/Serialization/Scrubbers/VerifySettings_ExtensionMappedInstanceScrubbers.cs +++ b/src/Verify/Serialization/Scrubbers/VerifySettings_ExtensionMappedInstanceScrubbers.cs @@ -9,7 +9,7 @@ public partial class VerifySettings /// /// Add a that applies to verified files with a matching extension. /// - public void AddScrubber(string extension, Scrubber scrubber) + internal void AddScrubber(string extension, Scrubber scrubber) { Ensure.NotNull(scrubber); ExtensionMappedInstanceSpanScrubbers ??= []; diff --git a/src/Verify/Serialization/Scrubbers/VerifySettings_InstanceScrubbers.cs b/src/Verify/Serialization/Scrubbers/VerifySettings_InstanceScrubbers.cs index de73a6cdf7..0f1f7d6afb 100644 --- a/src/Verify/Serialization/Scrubbers/VerifySettings_InstanceScrubbers.cs +++ b/src/Verify/Serialization/Scrubbers/VerifySettings_InstanceScrubbers.cs @@ -13,7 +13,7 @@ public partial class VerifySettings /// /// Add a . /// - public void AddScrubber(Scrubber scrubber) + internal void AddScrubber(Scrubber scrubber) { Ensure.NotNull(scrubber); InstanceSpanScrubbers ??= []; diff --git a/src/Verify/SettingsTask_Scrubbing.cs b/src/Verify/SettingsTask_Scrubbing.cs index 231df979dc..6de2afef66 100644 --- a/src/Verify/SettingsTask_Scrubbing.cs +++ b/src/Verify/SettingsTask_Scrubbing.cs @@ -12,7 +12,7 @@ public SettingsTask DisableScrubbers() /// [Pure] - public SettingsTask AddScrubber(Scrubber scrubber) + internal SettingsTask AddScrubber(Scrubber scrubber) { CurrentSettings.AddScrubber(scrubber); return this; @@ -20,7 +20,7 @@ public SettingsTask AddScrubber(Scrubber scrubber) /// [Pure] - public SettingsTask AddScrubber(string extension, Scrubber scrubber) + internal SettingsTask AddScrubber(string extension, Scrubber scrubber) { CurrentSettings.AddScrubber(extension, scrubber); return this; From f99eb5cb6bfb40e2816f869acd330a99fd53e6d5 Mon Sep 17 00:00:00 2001 From: Simon Cropp Date: Sun, 19 Jul 2026 21:53:44 +1000 Subject: [PATCH 32/33] Update Scrubber.cs --- src/Verify/Serialization/Scrubbers/Scrubber.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Verify/Serialization/Scrubbers/Scrubber.cs b/src/Verify/Serialization/Scrubbers/Scrubber.cs index 84026d7db4..9428def291 100644 --- a/src/Verify/Serialization/Scrubbers/Scrubber.cs +++ b/src/Verify/Serialization/Scrubbers/Scrubber.cs @@ -43,10 +43,10 @@ sealed class Scrubber // When set, the instance actually run is resolved at scrub time. Used by the // built-in date scrubbers, whose parse culture, window bounds, and anchor all // depend on the culture in effect when the scrub runs. - internal Func? Resolver { get; } + Func? resolver; internal Scrubber Resolve() => - Resolver?.Invoke() ?? this; + resolver?.Invoke() ?? this; Scrubber( ScrubberKind kind, @@ -63,7 +63,7 @@ internal Scrubber Resolve() => LineReplace? lineReplacer = null, Func? lineStringReplacer = null, WindowAnchor anchor = WindowAnchor.None, - char anchorChar = default, + char anchorChar = '\0', int anchorOffset = 0, Func? gate = null, Func? resolver = null) @@ -85,7 +85,7 @@ internal Scrubber Resolve() => AnchorChar = anchorChar; AnchorOffset = anchorOffset; Gate = gate; - Resolver = resolver; + this.resolver = resolver; } internal bool IsLineDrop => @@ -199,7 +199,7 @@ internal static Scrubber GatedWindow( Func gate, bool requireWordBoundary = false, WindowAnchor anchor = WindowAnchor.None, - char anchorChar = default, + char anchorChar = '\0', int anchorOffset = 0, Func? resolver = null) { From 6a8ed942d51d3c81af9d0a10e4bfdcad7ec39817 Mon Sep 17 00:00:00 2001 From: Simon Cropp Date: Sun, 19 Jul 2026 21:56:40 +1000 Subject: [PATCH 33/33] . --- .../Scrubbers/VerifierSettings_ExtensionMappedGlobalScrubbers.cs | 1 - .../Serialization/Scrubbers/VerifierSettings_GlobalScrubbers.cs | 1 - .../Scrubbers/VerifySettings_ExtensionMappedInstanceScrubbers.cs | 1 - .../Serialization/Scrubbers/VerifySettings_InstanceScrubbers.cs | 1 - 4 files changed, 4 deletions(-) diff --git a/src/Verify/Serialization/Scrubbers/VerifierSettings_ExtensionMappedGlobalScrubbers.cs b/src/Verify/Serialization/Scrubbers/VerifierSettings_ExtensionMappedGlobalScrubbers.cs index 2c17decd9e..60085999e4 100644 --- a/src/Verify/Serialization/Scrubbers/VerifierSettings_ExtensionMappedGlobalScrubbers.cs +++ b/src/Verify/Serialization/Scrubbers/VerifierSettings_ExtensionMappedGlobalScrubbers.cs @@ -12,7 +12,6 @@ public static partial class VerifierSettings internal static void AddScrubber(string extension, Scrubber scrubber) { InnerVerifier.ThrowIfVerifyHasBeenRun(); - Ensure.NotNull(scrubber); if (!ExtensionMappedGlobalSpanScrubbers.TryGetValue(extension, out var values)) { ExtensionMappedGlobalSpanScrubbers[extension] = values = []; diff --git a/src/Verify/Serialization/Scrubbers/VerifierSettings_GlobalScrubbers.cs b/src/Verify/Serialization/Scrubbers/VerifierSettings_GlobalScrubbers.cs index 2256554fa6..ea66c5860a 100644 --- a/src/Verify/Serialization/Scrubbers/VerifierSettings_GlobalScrubbers.cs +++ b/src/Verify/Serialization/Scrubbers/VerifierSettings_GlobalScrubbers.cs @@ -12,7 +12,6 @@ public static partial class VerifierSettings internal static void AddScrubber(Scrubber scrubber) { InnerVerifier.ThrowIfVerifyHasBeenRun(); - Ensure.NotNull(scrubber); GlobalSpanScrubbers.Add(scrubber); EngineScrubberSet.InvalidateGlobalCache(); } diff --git a/src/Verify/Serialization/Scrubbers/VerifySettings_ExtensionMappedInstanceScrubbers.cs b/src/Verify/Serialization/Scrubbers/VerifySettings_ExtensionMappedInstanceScrubbers.cs index db9fdb6f94..199a20841e 100644 --- a/src/Verify/Serialization/Scrubbers/VerifySettings_ExtensionMappedInstanceScrubbers.cs +++ b/src/Verify/Serialization/Scrubbers/VerifySettings_ExtensionMappedInstanceScrubbers.cs @@ -11,7 +11,6 @@ public partial class VerifySettings /// internal void AddScrubber(string extension, Scrubber scrubber) { - Ensure.NotNull(scrubber); ExtensionMappedInstanceSpanScrubbers ??= []; if (!ExtensionMappedInstanceSpanScrubbers.TryGetValue(extension, out var values)) diff --git a/src/Verify/Serialization/Scrubbers/VerifySettings_InstanceScrubbers.cs b/src/Verify/Serialization/Scrubbers/VerifySettings_InstanceScrubbers.cs index 0f1f7d6afb..281a099fab 100644 --- a/src/Verify/Serialization/Scrubbers/VerifySettings_InstanceScrubbers.cs +++ b/src/Verify/Serialization/Scrubbers/VerifySettings_InstanceScrubbers.cs @@ -15,7 +15,6 @@ public partial class VerifySettings /// internal void AddScrubber(Scrubber scrubber) { - Ensure.NotNull(scrubber); InstanceSpanScrubbers ??= []; InstanceSpanScrubbers.Add(scrubber); PropertyValueSetCache = null;