Skip to content

Normalization APIs using the spans #110465

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Dec 9, 2024
Merged

Normalization APIs using the spans #110465

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
808cb79
Normalization APIs using the spans
tarekgh Dec 6, 2024
d48e358
Address the feedback
tarekgh Dec 6, 2024
c6147e3
Update src/libraries/System.Private.CoreLib/src/System/Globalization/…
tarekgh Dec 8, 2024
cc8cace
Fix comment indent
ericstj Dec 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
124 changes: 114 additions & 10 deletions src/libraries/System.Private.CoreLib/src/System/Globalization/Normalization.Icu.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ private static unsafe bool IcuIsNormalized(string strInput, NormalizationForm no
{
Debug.Assert(!GlobalizationMode.Invariant);
Debug.Assert(!GlobalizationMode.UseNls);
Debug.Assert(normalizationForm == NormalizationForm.FormC || normalizationForm == NormalizationForm.FormD || normalizationForm == NormalizationForm.FormKC || normalizationForm == NormalizationForm.FormKD);

ValidateArguments(strInput, normalizationForm);

Expand All @@ -40,10 +41,43 @@ private static unsafe bool IcuIsNormalized(string strInput, NormalizationForm no
return ret == 1;
}

private static unsafe bool IcuIsNormalized(ReadOnlySpan<char> source, NormalizationForm normalizationForm)
{
Debug.Assert(!GlobalizationMode.Invariant);
Debug.Assert(!GlobalizationMode.UseNls);
Debug.Assert(!source.IsEmpty);
Debug.Assert(normalizationForm is NormalizationForm.FormC or NormalizationForm.FormD or NormalizationForm.FormKC or NormalizationForm.FormKD);

ValidateArguments(source, normalizationForm, nameof(source));

int ret;
fixed (char* pInput = source)
{
#if TARGET_MACCATALYST || TARGET_IOS || TARGET_TVOS
if (GlobalizationMode.Hybrid)
{
ret = Interop.Globalization.IsNormalizedNative(normalizationForm, pInput, source.Length);
}
else
#endif
{
ret = Interop.Globalization.IsNormalized(normalizationForm, pInput, source.Length);
}
}

if (ret == -1)
{
throw new ArgumentException(SR.Argument_InvalidCharSequenceNoIndex, nameof(source));
}

return ret == 1;
}

private static unsafe string IcuNormalize(string strInput, NormalizationForm normalizationForm)
{
Debug.Assert(!GlobalizationMode.Invariant);
Debug.Assert(!GlobalizationMode.UseNls);
Debug.Assert(normalizationForm == NormalizationForm.FormC || normalizationForm == NormalizationForm.FormD || normalizationForm == NormalizationForm.FormKC || normalizationForm == NormalizationForm.FormKD);

ValidateArguments(strInput, normalizationForm);

Expand Down Expand Up @@ -114,25 +148,95 @@ private static unsafe string IcuNormalize(string strInput, NormalizationForm nor
}
}

private static void ValidateArguments(string strInput, NormalizationForm normalizationForm)
private static unsafe bool IcuTryNormalize(ReadOnlySpan<char> source, Span<char> destination, out int charsWritten, NormalizationForm normalizationForm = NormalizationForm.FormC)
{
Debug.Assert(strInput != null);
Debug.Assert(!GlobalizationMode.Invariant);
Debug.Assert(!GlobalizationMode.UseNls);
Debug.Assert(!source.IsEmpty);
Debug.Assert(normalizationForm == NormalizationForm.FormC || normalizationForm == NormalizationForm.FormD || normalizationForm == NormalizationForm.FormKC || normalizationForm == NormalizationForm.FormKD);

if ((OperatingSystem.IsBrowser() || OperatingSystem.IsWasi())&& (normalizationForm == NormalizationForm.FormKC || normalizationForm == NormalizationForm.FormKD))
if (destination.IsEmpty)
{
// Browser's ICU doesn't contain data needed for FormKC and FormKD
throw new PlatformNotSupportedException();
charsWritten = 0;
return false;
}

ValidateArguments(source, normalizationForm, nameof(source));

int realLen;
fixed (char* pInput = source)
fixed (char* pDest = destination)
{
#if TARGET_MACCATALYST || TARGET_IOS || TARGET_TVOS
if (GlobalizationMode.Hybrid)
{
realLen = Interop.Globalization.NormalizeStringNative(normalizationForm, pInput, source.Length, pDest, destination.Length);
}
else
#endif
{
realLen = Interop.Globalization.NormalizeString(normalizationForm, pInput, source.Length, pDest, destination.Length);
}
}

if (realLen == -1)
{
throw new ArgumentException(SR.Argument_InvalidCharSequenceNoIndex, nameof(source));
}

if (normalizationForm != NormalizationForm.FormC && normalizationForm != NormalizationForm.FormD &&
normalizationForm != NormalizationForm.FormKC && normalizationForm != NormalizationForm.FormKD)
if (realLen <= destination.Length)
{
throw new ArgumentException(SR.Argument_InvalidNormalizationForm, nameof(normalizationForm));
charsWritten = realLen;
return true;
}

charsWritten = 0;
return false;
}

private static unsafe int IcuGetNormalizedLength(ReadOnlySpan<char> source, NormalizationForm normalizationForm)
{
Debug.Assert(!GlobalizationMode.Invariant);
Debug.Assert(!GlobalizationMode.UseNls);
Debug.Assert(!source.IsEmpty);
Debug.Assert(normalizationForm == NormalizationForm.FormC || normalizationForm == NormalizationForm.FormD || normalizationForm == NormalizationForm.FormKC || normalizationForm == NormalizationForm.FormKD);

ValidateArguments(source, normalizationForm, nameof(source));

int realLen;
fixed (char* pInput = source)
{
#if TARGET_MACCATALYST || TARGET_IOS || TARGET_TVOS
if (GlobalizationMode.Hybrid)
{
realLen = Interop.Globalization.NormalizeStringNative(normalizationForm, pInput, source.Length, null, 0);
}
else
#endif
{
realLen = Interop.Globalization.NormalizeString(normalizationForm, pInput, source.Length, null, 0);
}
}

if (realLen < 0)
{
throw new ArgumentException(SR.Argument_InvalidCharSequenceNoIndex, nameof(source));
}

return realLen;
}

private static void ValidateArguments(ReadOnlySpan<char> strInput, NormalizationForm normalizationForm, string paramName = "strInput")
{
if ((OperatingSystem.IsBrowser() || OperatingSystem.IsWasi()) && (normalizationForm == NormalizationForm.FormKC || normalizationForm == NormalizationForm.FormKD))
{
// Browser's ICU doesn't contain data needed for FormKC and FormKD
throw new PlatformNotSupportedException();
}

if (HasInvalidUnicodeSequence(strInput))
{
throw new ArgumentException(SR.Argument_InvalidCharSequenceNoIndex, nameof(strInput));
throw new ArgumentException(SR.Argument_InvalidCharSequenceNoIndex, paramName);
}
}

Expand All @@ -143,7 +247,7 @@ private static void ValidateArguments(string strInput, NormalizationForm normali
/// We walk the string ourselves looking for these bad sequences so we can continue to throw
/// ArgumentException in these cases.
/// </summary>
private static bool HasInvalidUnicodeSequence(string s)
private static bool HasInvalidUnicodeSequence(ReadOnlySpan<char> s)
{
for (int i = 0; i < s.Length; i++)
{
Expand Down
169 changes: 135 additions & 34 deletions src/libraries/System.Private.CoreLib/src/System/Globalization/Normalization.Nls.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,52 +3,52 @@

using System.Buffers;
using System.Diagnostics;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Text;

namespace System.Globalization
{
internal static partial class Normalization
{
private static unsafe bool NlsIsNormalized(string strInput, NormalizationForm normalizationForm)
private static unsafe bool NlsIsNormalized(ReadOnlySpan<char> source, NormalizationForm normalizationForm)
{
Debug.Assert(!GlobalizationMode.Invariant);
Debug.Assert(GlobalizationMode.UseNls);
Debug.Assert(strInput != null);
Debug.Assert(!source.IsEmpty);
Debug.Assert(normalizationForm == NormalizationForm.FormC || normalizationForm == NormalizationForm.FormD || normalizationForm == NormalizationForm.FormKC || normalizationForm == NormalizationForm.FormKD);

Interop.BOOL result;
fixed (char* pInput = source)
{
result = Interop.Normaliz.IsNormalizedString(normalizationForm, pInput, source.Length);
}

// The only way to know if IsNormalizedString failed is through checking the Win32 last error
// IsNormalizedString pinvoke has SetLastError attribute property which will set the last error
// to 0 (ERROR_SUCCESS) before executing the calls.
CheckLastErrorAndThrowIfFailed(nameof(source));

return result != Interop.BOOL.FALSE;
}

private static unsafe bool NlsIsNormalized(string strInput, NormalizationForm normalizationForm)
{
Debug.Assert(!GlobalizationMode.Invariant);
Debug.Assert(GlobalizationMode.UseNls);
Debug.Assert(strInput != null);
Debug.Assert(normalizationForm == NormalizationForm.FormC || normalizationForm == NormalizationForm.FormD || normalizationForm == NormalizationForm.FormKC || normalizationForm == NormalizationForm.FormKD);

Interop.BOOL result;
fixed (char* pInput = strInput)
{
result = Interop.Normaliz.IsNormalizedString(normalizationForm, pInput, strInput.Length);
}

int lastError = Marshal.GetLastPInvokeError();
switch (lastError)
{
case Interop.Errors.ERROR_SUCCESS:
break;

case Interop.Errors.ERROR_INVALID_PARAMETER:
case Interop.Errors.ERROR_NO_UNICODE_TRANSLATION:
if (normalizationForm != NormalizationForm.FormC &&
normalizationForm != NormalizationForm.FormD &&
normalizationForm != NormalizationForm.FormKC &&
normalizationForm != NormalizationForm.FormKD)
{
throw new ArgumentException(SR.Argument_InvalidNormalizationForm, nameof(normalizationForm));
}

throw new ArgumentException(SR.Argument_InvalidCharSequenceNoIndex, nameof(strInput));

case Interop.Errors.ERROR_NOT_ENOUGH_MEMORY:
throw new OutOfMemoryException();

default:
throw new InvalidOperationException(SR.Format(SR.UnknownError_Num, lastError));
}
// The only way to know if IsNormalizedString failed is through checking the Win32 last error
// IsNormalizedString pinvoke has SetLastError attribute property which will set the last error
// to 0 (ERROR_SUCCESS) before executing the calls.
CheckLastErrorAndThrowIfFailed(nameof(strInput));

return result != Interop.BOOL.FALSE;
}
Expand All @@ -58,6 +58,7 @@ private static unsafe string NlsNormalize(string strInput, NormalizationForm nor
Debug.Assert(!GlobalizationMode.Invariant);
Debug.Assert(GlobalizationMode.UseNls);
Debug.Assert(strInput != null);
Debug.Assert(normalizationForm == NormalizationForm.FormC || normalizationForm == NormalizationForm.FormD || normalizationForm == NormalizationForm.FormKC || normalizationForm == NormalizationForm.FormKD);

if (strInput.Length == 0)
{
Expand Down Expand Up @@ -111,14 +112,6 @@ private static unsafe string NlsNormalize(string strInput, NormalizationForm nor

case Interop.Errors.ERROR_INVALID_PARAMETER:
case Interop.Errors.ERROR_NO_UNICODE_TRANSLATION:
if (normalizationForm != NormalizationForm.FormC &&
normalizationForm != NormalizationForm.FormD &&
normalizationForm != NormalizationForm.FormKC &&
normalizationForm != NormalizationForm.FormKD)
{
throw new ArgumentException(SR.Argument_InvalidNormalizationForm, nameof(normalizationForm));
}

// Illegal code point or order found. Ie: FFFE or D800 D800, etc.
throw new ArgumentException(SR.Argument_InvalidCharSequenceNoIndex, nameof(strInput));

Expand All @@ -139,5 +132,113 @@ private static unsafe string NlsNormalize(string strInput, NormalizationForm nor
}
}
}

private static unsafe bool NlsTryNormalize(ReadOnlySpan<char> source, Span<char> destination, out int charsWritten, NormalizationForm normalizationForm = NormalizationForm.FormC)
{
Debug.Assert(!GlobalizationMode.Invariant);
Debug.Assert(GlobalizationMode.UseNls);
Debug.Assert(!source.IsEmpty);
Debug.Assert(normalizationForm == NormalizationForm.FormC || normalizationForm == NormalizationForm.FormD || normalizationForm == NormalizationForm.FormKC || normalizationForm == NormalizationForm.FormKD);

if (destination.IsEmpty)
{
charsWritten = 0;
return false;
}

// we depend on Win32 last error when calling NormalizeString
// NormalizeString pinvoke has SetLastError attribute property which will set the last error
// to 0 (ERROR_SUCCESS) before executing the calls.

int realLength;
fixed (char* pInput = source)
fixed (char* pDest = destination)
{
realLength = Interop.Normaliz.NormalizeString(normalizationForm, pInput, source.Length, pDest, destination.Length);
}

int lastError = Marshal.GetLastPInvokeError();
switch (lastError)
{
case Interop.Errors.ERROR_SUCCESS:
charsWritten = realLength;
return true;

// Do appropriate stuff for the individual errors:
case Interop.Errors.ERROR_INSUFFICIENT_BUFFER:
charsWritten = 0;
return false;

case Interop.Errors.ERROR_INVALID_PARAMETER:
case Interop.Errors.ERROR_NO_UNICODE_TRANSLATION:
// Illegal code point or order found. Ie: FFFE or D800 D800, etc.
throw new ArgumentException(SR.Argument_InvalidCharSequenceNoIndex, nameof(source));

case Interop.Errors.ERROR_NOT_ENOUGH_MEMORY:
throw new OutOfMemoryException();

default:
// We shouldn't get here...
throw new InvalidOperationException(SR.Format(SR.UnknownError_Num, lastError));
}
}

private static unsafe int NlsGetNormalizedLength(ReadOnlySpan<char> source, NormalizationForm normalizationForm)
{
Debug.Assert(!GlobalizationMode.Invariant);
Debug.Assert(GlobalizationMode.UseNls);
Debug.Assert(!source.IsEmpty);
Debug.Assert(normalizationForm == NormalizationForm.FormC || normalizationForm == NormalizationForm.FormD || normalizationForm == NormalizationForm.FormKC || normalizationForm == NormalizationForm.FormKD);

// we depend on Win32 last error when calling NormalizeString
// NormalizeString pinvoke has SetLastError attribute property which will set the last error
// to 0 (ERROR_SUCCESS) before executing the calls.

int realLength;
fixed (char* pInput = source)
{
realLength = Interop.Normaliz.NormalizeString(normalizationForm, pInput, source.Length, null, 0);
}

int lastError = Marshal.GetLastPInvokeError();
switch (lastError)
{
case Interop.Errors.ERROR_SUCCESS:
return realLength;

case Interop.Errors.ERROR_INVALID_PARAMETER:
case Interop.Errors.ERROR_NO_UNICODE_TRANSLATION:
// Illegal code point or order found. Ie: FFFE or D800 D800, etc.
throw new ArgumentException(SR.Argument_InvalidCharSequenceNoIndex, nameof(source));

case Interop.Errors.ERROR_NOT_ENOUGH_MEMORY:
throw new OutOfMemoryException();

default:
// We shouldn't get here...
throw new InvalidOperationException(SR.Format(SR.UnknownError_Num, lastError));
}
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void CheckLastErrorAndThrowIfFailed(string inputName)
{
int lastError = Marshal.GetLastPInvokeError();
switch (lastError)
{
case Interop.Errors.ERROR_SUCCESS:
break;

case Interop.Errors.ERROR_INVALID_PARAMETER:
case Interop.Errors.ERROR_NO_UNICODE_TRANSLATION:
throw new ArgumentException(SR.Argument_InvalidCharSequenceNoIndex, inputName);

case Interop.Errors.ERROR_NOT_ENOUGH_MEMORY:
throw new OutOfMemoryException();

default:
throw new InvalidOperationException(SR.Format(SR.UnknownError_Num, lastError));
}
}
}
}
Loading
Loading