From 0491b672ce1e7892641914d4416c45f8c40fad70 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 7 Oct 2026 11:40:09 +0000 Subject: [PATCH] Bound persistence file names in UTF-8 bytes as well as characters [patch] GetSafeFileName truncated only past 100 UTF-16 characters, but Linux and macOS limit a file-name component to 255 bytes. A key of 83+ CJK characters stayed untruncated, and StoreAsync threw PathTooLongException once the extension and ".tmp" were appended. Names are now also truncated past 246 UTF-8 bytes, which leaves room for a five-byte extension and ".tmp". Every name that could be written before stays verbatim, so existing files are still found. Truncation takes whole code points, so a surrogate pair is no longer split, and IsTruncatedName recognises names cut at the byte budget. Fixes #48 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01D7wytU6jsZ1cH3f6grCTz5 --- Essentials.Tests/PersistenceNamingTests.cs | 75 +++++++++++++++++++++- Essentials/PersistenceProviderUtilities.cs | 51 +++++++++++++-- 2 files changed, 118 insertions(+), 8 deletions(-) diff --git a/Essentials.Tests/PersistenceNamingTests.cs b/Essentials.Tests/PersistenceNamingTests.cs index 67e14d6..9b59244 100644 --- a/Essentials.Tests/PersistenceNamingTests.cs +++ b/Essentials.Tests/PersistenceNamingTests.cs @@ -1,4 +1,4 @@ -// Copyright (c) 2023-2026 ktsu-dev contributors +// Copyright (c) 2023-2026 ktsu-dev contributors namespace ktsu.Essentials.Tests; @@ -7,6 +7,7 @@ namespace ktsu.Essentials.Tests; using System.Globalization; using System.IO; using System.Linq; +using System.Text; using ktsu.Essentials; using ktsu.Essentials.FileSystemProviders.Native; using ktsu.Essentials.PersistenceProviders.FileSystem; @@ -88,6 +89,78 @@ public void SafeFileName_Bounds_Length_And_Stays_Distinct() Assert.IsNull(PersistenceProviderUtilities.GetKeyFromFileName(first), "Truncated names are not recoverable and must report so"); } + /// + /// Keys whose encoded names exceed the 255-byte file-name limit of Linux and macOS while staying within + /// 100 UTF-16 characters, or whose truncation point falls inside a surrogate pair. See issue #48. + /// + public static IEnumerable MultibyteKeys => + [ + [new string('日', 90)], + [new string('日', 83)], + [string.Concat(Enumerable.Repeat("😀", 60))], + ["a" + string.Concat(Enumerable.Repeat("😀", 60))], + [string.Concat(Enumerable.Repeat("é日😀%", 30))], + ]; + + [TestMethod] + [DynamicData(nameof(MultibyteKeys))] + public void SafeFileName_Bounds_Utf8_Bytes_Without_Splitting_A_Code_Point(string key) + { + string encoded = PersistenceProviderUtilities.GetSafeFileName(key); + + // The longest suffix the file-system providers append is a five-byte extension and ".tmp". + Assert.IsLessThanOrEqualTo(255 - 9, Encoding.UTF8.GetByteCount(encoded), "The name plus its extension and .tmp must fit in 255 bytes"); + for (int i = 0; i < encoded.Length; i++) + { + if (char.IsHighSurrogate(encoded[i])) + { + Assert.IsTrue(i + 1 < encoded.Length && char.IsLowSurrogate(encoded[i + 1]), $"A surrogate pair was split at {i}"); + i++; + } + else + { + Assert.IsFalse(char.IsLowSurrogate(encoded[i]), $"A lone low surrogate was left at {i}"); + } + } + + Assert.IsNull(PersistenceProviderUtilities.GetKeyFromFileName(encoded), "A truncated name must be reported as unrecoverable"); + Assert.AreNotEqual(encoded, PersistenceProviderUtilities.GetSafeFileName(key + "x"), "Truncated keys must remain distinct"); + } + + [TestMethod] + [DynamicData(nameof(MultibyteKeys))] + public async Task Long_Multibyte_Keys_Store_And_Retrieve_On_The_Native_File_System(string key) + { + string dir = Directory.CreateTempSubdirectory("NamingTests_").FullName; + try + { + FileSystemPersistenceProvider persistence = new(new NativeFileSystemProvider(), new YamlSerializationProvider(), dir); + + await persistence.StoreAsync(key, "value", TestContext.CancellationToken).ConfigureAwait(false); + + Assert.IsTrue(await persistence.ExistsAsync(key, TestContext.CancellationToken).ConfigureAwait(false), "A stored key must exist"); + Assert.AreEqual("value", await persistence.RetrieveAsync(key, TestContext.CancellationToken).ConfigureAwait(false), "A stored key must read back"); + } + finally + { + if (Directory.Exists(dir)) + { + Directory.Delete(dir, true); + } + } + } + + [TestMethod] + public void SafeFileName_Keeps_Names_That_Fit_The_Byte_Budget_Verbatim() + { + // 82 CJK characters encode to 246 bytes, the most a name can be and still be written with a + // five-byte extension and ".tmp". Names like it were stored before the byte budget existed, so + // they must keep resolving to the same file. + string key = new('日', 82); + + Assert.AreEqual(key, PersistenceProviderUtilities.GetSafeFileName(key), "A name within the byte budget must not be truncated"); + } + [TestMethod] public async Task Keys_Containing_A_Tilde_Are_Listed() { diff --git a/Essentials/PersistenceProviderUtilities.cs b/Essentials/PersistenceProviderUtilities.cs index 41be976..7e383ff 100644 --- a/Essentials/PersistenceProviderUtilities.cs +++ b/Essentials/PersistenceProviderUtilities.cs @@ -1,4 +1,4 @@ -// Copyright (c) 2023-2026 ktsu-dev contributors +// Copyright (c) 2023-2026 ktsu-dev contributors namespace ktsu.Essentials; @@ -32,6 +32,16 @@ public static class PersistenceProviderUtilities /// private const int MaxEncodedLength = 100; + /// + /// The most UTF-8 bytes an encoded name is kept verbatim at. Linux and macOS limit a file-name component + /// to 255 bytes, not UTF-16 characters, so a name of multibyte characters can be under + /// and still too long. The file-system providers append the serializer's + /// extension and then .tmp, so this leaves room for both with a five-byte extension, the length of + /// every built-in serializer's. Every name that could be written before this limit existed is therefore + /// kept, and still finds its file. See issue #48. + /// + private const int MaxEncodedBytes = 255 - 9; + /// /// Separates the truncated prefix from its hash. Keys may contain it too, so a truncated name is /// recognised by its whole shape (see ), not by the marker alone. @@ -89,7 +99,9 @@ public static string GetSafeFileName(string input) } string encoded = builder.ToString(); - return encoded.Length > MaxEncodedLength ? Truncate(encoded) : encoded; + return encoded.Length > MaxEncodedLength || Encoding.UTF8.GetByteCount(encoded) > MaxEncodedBytes + ? Truncate(encoded) + : encoded; } /// @@ -288,7 +300,24 @@ private static bool IsReservedDeviceName(string value) private static string Truncate(string encoded) { string hash = Fnv1a64(encoded).ToString("x16", CultureInfo.InvariantCulture); - int prefixLength = MaxEncodedLength - hash.Length - 1; + int maxPrefixLength = MaxEncodedLength - hash.Length - 1; + int maxPrefixBytes = MaxEncodedBytes - hash.Length - 1; + + // Take whole code points while both budgets allow, so a surrogate pair is never split. + int prefixLength = 0; + int prefixBytes = 0; + while (prefixLength < encoded.Length) + { + int width = char.IsSurrogatePair(encoded, prefixLength) ? 2 : 1; + int bytes = Encoding.UTF8.GetByteCount(encoded.AsSpan(prefixLength, width)); + if (prefixLength + width > maxPrefixLength || prefixBytes + bytes > maxPrefixBytes) + { + break; + } + + prefixLength += width; + prefixBytes += bytes; + } // Never split a percent-escape across the truncation boundary. while (prefixLength > 0 && IsInsideEscape(encoded, prefixLength)) @@ -301,7 +330,8 @@ private static string Truncate(string encoded) /// /// Reports whether has the shape produces: at or - /// just under , ending in the marker and a lowercase hexadecimal hash. + /// just under characters or UTF-8 bytes, + /// ending in the marker and a lowercase hexadecimal hash. /// /// /// Checking only for the marker hid every key that merely contains a ~, such as @@ -309,10 +339,17 @@ private static string Truncate(string encoded) /// private static bool IsTruncatedName(string fileName) { - // Truncate backs off by up to two characters to avoid splitting a percent-escape. - if (fileName.Length is < (MaxEncodedLength - 2) or > MaxEncodedLength) + // Truncate stops up to one character short of the character budget rather than split a surrogate + // pair, or up to three bytes short of the byte budget rather than split a code point, and then backs + // off by up to two more characters, each one byte, to avoid splitting a percent-escape. + bool nearMaxLength = fileName.Length is >= (MaxEncodedLength - 3) and <= MaxEncodedLength; + if (!nearMaxLength) { - return false; + int byteCount = Encoding.UTF8.GetByteCount(fileName); + if (byteCount is < (MaxEncodedBytes - 5) or > MaxEncodedBytes) + { + return false; + } } int markerIndex = fileName.Length - TruncationHashLength - 1;