Bound persistence file names in UTF-8 bytes as well as characters [patch]

Essentials.Tests/PersistenceNamingTests.cs

@@ -1,4 +1,4 @@
-// Copyright (c) 2023-2026 ktsu-dev contributors
+// Copyright (c) 2023-2026 ktsu-dev contributors
 
 namespace ktsu.Essentials.Tests;
 
@@ -7,6 +7,7 @@ namespace ktsu.Essentials.Tests;
 using System.Globalization;
 using System.IO;
 using System.Linq;
+using System.Text;
 using ktsu.Essentials;
 using ktsu.Essentials.FileSystemProviders.Native;
 using ktsu.Essentials.PersistenceProviders.FileSystem;
@@ -88,6 +89,78 @@ public void SafeFileName_Bounds_Length_And_Stays_Distinct()
 		Assert.IsNull(PersistenceProviderUtilities.GetKeyFromFileName(first), "Truncated names are not recoverable and must report so");
 	}
 
+	/// <summary>
+	/// Keys whose encoded names exceed the 255-byte file-name limit of Linux and macOS while staying within
+	/// 100 UTF-16 characters, or whose truncation point falls inside a surrogate pair. See issue #48.
+	/// </summary>
+	public static IEnumerable<object[]> MultibyteKeys =>
+	[
+		[new string('日', 90)],
+		[new string('日', 83)],
+		[string.Concat(Enumerable.Repeat("😀", 60))],
+		["a" + string.Concat(Enumerable.Repeat("😀", 60))],
+		[string.Concat(Enumerable.Repeat("é日😀%", 30))],
+	];
+
+	[TestMethod]
+	[DynamicData(nameof(MultibyteKeys))]
+	public void SafeFileName_Bounds_Utf8_Bytes_Without_Splitting_A_Code_Point(string key)
+	{
+		string encoded = PersistenceProviderUtilities.GetSafeFileName(key);
+
+		// The longest suffix the file-system providers append is a five-byte extension and ".tmp".
+		Assert.IsLessThanOrEqualTo(255 - 9, Encoding.UTF8.GetByteCount(encoded), "The name plus its extension and .tmp must fit in 255 bytes");
+		for (int i = 0; i < encoded.Length; i++)
+		{
+			if (char.IsHighSurrogate(encoded[i]))
+			{
+				Assert.IsTrue(i + 1 < encoded.Length && char.IsLowSurrogate(encoded[i + 1]), $"A surrogate pair was split at {i}");
+				i++;
+			}
+			else
+			{
+				Assert.IsFalse(char.IsLowSurrogate(encoded[i]), $"A lone low surrogate was left at {i}");
+			}
+		}
+
+		Assert.IsNull(PersistenceProviderUtilities.GetKeyFromFileName(encoded), "A truncated name must be reported as unrecoverable");
+		Assert.AreNotEqual(encoded, PersistenceProviderUtilities.GetSafeFileName(key + "x"), "Truncated keys must remain distinct");
+	}
+
+	[TestMethod]
+	[DynamicData(nameof(MultibyteKeys))]
+	public async Task Long_Multibyte_Keys_Store_And_Retrieve_On_The_Native_File_System(string key)
+	{
+		string dir = Directory.CreateTempSubdirectory("NamingTests_").FullName;
+		try
+		{
+			FileSystemPersistenceProvider<string> persistence = new(new NativeFileSystemProvider(), new YamlSerializationProvider(), dir);
+
+			await persistence.StoreAsync(key, "value", TestContext.CancellationToken).ConfigureAwait(false);
+
+			Assert.IsTrue(await persistence.ExistsAsync(key, TestContext.CancellationToken).ConfigureAwait(false), "A stored key must exist");
+			Assert.AreEqual("value", await persistence.RetrieveAsync<string>(key, TestContext.CancellationToken).ConfigureAwait(false), "A stored key must read back");
+		}
+		finally
+		{
+			if (Directory.Exists(dir))
+			{
+				Directory.Delete(dir, true);
+			}
+		}
+	}
+
+	[TestMethod]
+	public void SafeFileName_Keeps_Names_That_Fit_The_Byte_Budget_Verbatim()
+	{
+		// 82 CJK characters encode to 246 bytes, the most a name can be and still be written with a
+		// five-byte extension and ".tmp". Names like it were stored before the byte budget existed, so
+		// they must keep resolving to the same file.
+		string key = new('日', 82);
+
+		Assert.AreEqual(key, PersistenceProviderUtilities.GetSafeFileName(key), "A name within the byte budget must not be truncated");
+	}
+
 	[TestMethod]
 	public async Task Keys_Containing_A_Tilde_Are_Listed()
 	{

Essentials/PersistenceProviderUtilities.cs

@@ -1,4 +1,4 @@
-// Copyright (c) 2023-2026 ktsu-dev contributors
+// Copyright (c) 2023-2026 ktsu-dev contributors
 
 namespace ktsu.Essentials;
 
@@ -32,6 +32,16 @@
 	/// </summary>
 	private const int MaxEncodedLength = 100;
 
+	/// <summary>
+	/// The most UTF-8 bytes an encoded name is kept verbatim at. Linux and macOS limit a file-name component
+	/// to 255 bytes, not UTF-16 characters, so a name of multibyte characters can be under
+	/// <see cref="MaxEncodedLength"/> and still too long. The file-system providers append the serializer's
+	/// extension and then <c>.tmp</c>, so this leaves room for both with a five-byte extension, the length of
+	/// every built-in serializer's. Every name that could be written before this limit existed is therefore
+	/// kept, and still finds its file. See issue #48.
+	/// </summary>
+	private const int MaxEncodedBytes = 255 - 9;
+
 	/// <summary>
 	/// Separates the truncated prefix from its hash. Keys may contain it too, so a truncated name is
 	/// recognised by its whole shape (see <see cref="IsTruncatedName"/>), not by the marker alone.
@@ -89,7 +99,9 @@
 		}
 
 		string encoded = builder.ToString();
-		return encoded.Length > MaxEncodedLength ? Truncate(encoded) : encoded;
+		return encoded.Length > MaxEncodedLength || Encoding.UTF8.GetByteCount(encoded) > MaxEncodedBytes
+			? Truncate(encoded)
+			: encoded;
 	}
 
 	/// <summary>
@@ -121,7 +133,7 @@
 			}
 
 			builder.Append((char)value);
 			i += 2;
 		}
 
 		return builder.ToString();
@@ -288,7 +300,24 @@
 	private static string Truncate(string encoded)
 	{
 		string hash = Fnv1a64(encoded).ToString("x16", CultureInfo.InvariantCulture);
-		int prefixLength = MaxEncodedLength - hash.Length - 1;
+		int maxPrefixLength = MaxEncodedLength - hash.Length - 1;
+		int maxPrefixBytes = MaxEncodedBytes - hash.Length - 1;
+
+		// Take whole code points while both budgets allow, so a surrogate pair is never split.
+		int prefixLength = 0;
+		int prefixBytes = 0;
+		while (prefixLength < encoded.Length)
+		{
+			int width = char.IsSurrogatePair(encoded, prefixLength) ? 2 : 1;
+			int bytes = Encoding.UTF8.GetByteCount(encoded.AsSpan(prefixLength, width));
+			if (prefixLength + width > maxPrefixLength || prefixBytes + bytes > maxPrefixBytes)
+			{
+				break;
+			}
+
+			prefixLength += width;
+			prefixBytes += bytes;
+		}
 
 		// Never split a percent-escape across the truncation boundary.
 		while (prefixLength > 0 && IsInsideEscape(encoded, prefixLength))
@@ -301,18 +330,26 @@
 
 	/// <summary>
 	/// Reports whether <paramref name="fileName"/> has the shape <see cref="Truncate"/> produces: at or
-	/// just under <see cref="MaxEncodedLength"/>, ending in the marker and a lowercase hexadecimal hash.
+	/// just under <see cref="MaxEncodedLength"/> characters or <see cref="MaxEncodedBytes"/> UTF-8 bytes,
+	/// ending in the marker and a lowercase hexadecimal hash.
 	/// </summary>
 	/// <remarks>
 	/// Checking only for the marker hid every key that merely contains a <c>~</c>, such as
 	/// <c>backup~1</c>, from key listings even though it was stored and retrievable.
 	/// </remarks>
 	private static bool IsTruncatedName(string fileName)
 	{
-		// Truncate backs off by up to two characters to avoid splitting a percent-escape.
-		if (fileName.Length is < (MaxEncodedLength - 2) or > MaxEncodedLength)
+		// Truncate stops up to one character short of the character budget rather than split a surrogate
+		// pair, or up to three bytes short of the byte budget rather than split a code point, and then backs
+		// off by up to two more characters, each one byte, to avoid splitting a percent-escape.
+		bool nearMaxLength = fileName.Length is >= (MaxEncodedLength - 3) and <= MaxEncodedLength;
+		if (!nearMaxLength)
 		{
-			return false;
+			int byteCount = Encoding.UTF8.GetByteCount(fileName);
+			if (byteCount is < (MaxEncodedBytes - 5) or > MaxEncodedBytes)
+			{
+				return false;
+			}
 		}
 
 		int markerIndex = fileName.Length - TruncationHashLength - 1;