Skip to content

Pack OrderBy ThenBy keys in LINQ #120786

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
+441 −3
Open

Pack OrderBy ThenBy keys in LINQ #120786

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
2b23a71
Packing keys on EnumerableSorter
henriquewr Oct 16, 2025
039b84a
Merge branch 'dotnet:main' into packedOrder
henriquewr Oct 29, 2025
2544a9a
Big endian impl
henriquewr Oct 29, 2025
a451af3
Merge branch 'packedOrder' of https://github.com/henriquewr/runtime i…
henriquewr Oct 29, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
306 changes: 303 additions & 3 deletions src/libraries/System.Linq/src/System/Linq/OrderedEnumerable.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,9 @@
using System.Collections;
using System.Collections.Generic;
using System.Diagnostics;
using System.Diagnostics.CodeAnalysis;
using System.Numerics;
using System.Runtime.CompilerServices;

namespace System.Linq
{
Expand Down Expand Up @@ -101,7 +104,17 @@ internal override EnumerableSorter<TElement> GetEnumerableSorter(EnumerableSorte
comparer = (IComparer<TKey>)StringComparer.CurrentCulture;
}

EnumerableSorter<TElement> sorter = new EnumerableSorter<TElement, TKey>(_keySelector, comparer, _descending, next);
EnumerableSorter<TElement> sorter;

if (next is null)
{
sorter = new EnumerableSorter<TElement, TKey>(_keySelector, comparer, _descending, next);
}
else
{
sorter = next.CreateWithChild(_keySelector, comparer, _descending);
}

if (_parent is not null)
{
sorter = _parent.GetEnumerableSorter(sorter);
Expand All @@ -122,7 +135,7 @@ public override bool MoveNext()
{
int state = _state;

Initialized:
Initialized:
if (state > 1)
{
Debug.Assert(_buffer is not null);
Expand Down Expand Up @@ -195,7 +208,7 @@ public override bool MoveNext()
int state = _state;
TElement[]? buffer;

Initialized:
Initialized:
if (state > 1)
{
buffer = _buffer;
Expand Down Expand Up @@ -343,6 +356,8 @@ private int[] ComputeMap(TElement[] elements, int count)
return map;
}

internal abstract EnumerableSorter<TElement> CreateWithChild<TChildKey>(Func<TElement, TChildKey> keySelector, IComparer<TChildKey> comparer, bool descending);

internal int[] Sort(TElement[] elements, int count)
{
int[] map = ComputeMap(elements, count);
Expand Down Expand Up @@ -385,6 +400,7 @@ private sealed class EnumerableSorter<TElement, TKey> : EnumerableSorter<TElemen
private readonly bool _descending;
private readonly EnumerableSorter<TElement>? _next;
private TKey[]? _keys;
private readonly byte _packingUsedSize;
Copy link
Author

@henriquewr henriquewr Oct 16, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The purpose of this field is saving the bytes used for the packing

For example: If you pack an int and a bool, only 5 bytes will be used
and the packing will use the ulong (8 bytes) type to accommodate everything

And still have 3 bytes left for some other type


internal EnumerableSorter(Func<TElement, TKey> keySelector, IComparer<TKey> comparer, bool descending, EnumerableSorter<TElement>? next)
{
Expand All @@ -394,6 +410,31 @@ internal EnumerableSorter(Func<TElement, TKey> keySelector, IComparer<TKey> comp
_next = next;
}

private EnumerableSorter(Func<TElement, TKey> keySelector, IComparer<TKey> comparer, bool descending, EnumerableSorter<TElement>? next, byte packingUsedSize)
{
_keySelector = keySelector;
_comparer = comparer;
_descending = descending;
_next = next;
_packingUsedSize = packingUsedSize;
}

internal override EnumerableSorter<TElement> CreateWithChild<TChildKey>(Func<TElement, TChildKey> keySelector, IComparer<TChildKey> comparer, bool descending)
{
EnumerableSorter<TElement> sorter;

if (TryPackKeys(keySelector, comparer, descending, this, out var packedSorter))
{
sorter = packedSorter;
}
else
{
sorter = new EnumerableSorter<TElement, TChildKey>(keySelector, comparer, descending, this);
}

return sorter;
}

internal override void ComputeKeys(TElement[] elements, int count)
{
Func<TElement, TKey> keySelector = _keySelector;
Expand Down Expand Up @@ -653,6 +694,265 @@ protected override int Min(int[] map, int count)
}
return map[index];
}

private static bool TryPackKeys<TKey1, TKey2>(Func<TElement, TKey1> keySelector, IComparer<TKey1> comparer, bool descending, EnumerableSorter<TElement, TKey2> next, [NotNullWhen(true)] out EnumerableSorter<TElement>? packedSorter)
{
bool key1typeIsSigned;
bool key2typeIsSigned;
int key1typeSize;
int key2typeSize;

if (!TryGetPackingTypeData<TKey1>(out key1typeIsSigned, out key1typeSize) ||
!TryGetPackingTypeData<TKey2>(out key2typeIsSigned, out key2typeSize) ||
comparer != Comparer<TKey1>.Default || next._comparer != Comparer<TKey2>.Default)
{
packedSorter = null;
return false;
}

int packingSize = next._packingUsedSize == 0 ? key2typeSize : next._packingUsedSize;

byte totalSize = (byte)(key1typeSize + packingSize);

int keyPadding = BitConverter.IsLittleEndian ? packingSize : key1typeSize;

if (totalSize <= sizeof(uint))
{
uint toggle = 0U;
if (key1typeIsSigned)
{
toggle |= 1U << ((totalSize * 8) - 1);
Copy link
Author

@henriquewr henriquewr Oct 16, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This makes the toggle for the sign bit:
1U << ((sizeof(sbyte) * 8) - 1)
would be in binary:
00000000_00000000_00000000_10000000

}
if (key2typeIsSigned)
{
toggle |= 1U << ((key2typeSize * 8) - 1);
}
packedSorter = CreatePacked<uint, TKey1, TKey2>(
highKeySelector: keySelector,
lowKeySelector: next._keySelector,
keyPadding: keyPadding,
totalSize: totalSize,
key1IsDescending: descending,
key2IsDescending: next._descending,
toggleSignBits: toggle,
next: next?._next
);
return true;
}

if (totalSize <= sizeof(ulong))
{
ulong toggle = 0UL;
if (key1typeIsSigned)
{
toggle |= 1UL << ((totalSize * 8) - 1);
}
if (key2typeIsSigned)
{
toggle |= 1UL << ((key2typeSize * 8) - 1);
}
packedSorter = CreatePacked<ulong, TKey1, TKey2>(
highKeySelector: keySelector,
lowKeySelector: next._keySelector,
keyPadding: keyPadding,
totalSize: totalSize,
key1IsDescending: descending,
key2IsDescending: next._descending,
toggleSignBits: toggle,
next: next?._next
);
return true;
}

packedSorter = null;
return false;
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryGetPackingTypeData<T>(out bool isSigned, out int size)
{
if (typeof(T) == typeof(sbyte))
{
isSigned = true;
size = sizeof(sbyte);
return true;
}
if (typeof(T) == typeof(short))
{
isSigned = true;
size = sizeof(short);
return true;
}
if (typeof(T) == typeof(int))
{
isSigned = true;
size = sizeof(int);
return true;
}
if (typeof(T) == typeof(nint))
{
isSigned = true;
size = Unsafe.SizeOf<nint>();
return true;
}
if (typeof(T) == typeof(long))
{
isSigned = true;
size = sizeof(long);
return true;
}

if (typeof(T) == typeof(byte))
{
isSigned = false;
size = sizeof(byte);
return true;
}
if (typeof(T) == typeof(ushort))
{
isSigned = false;
size = sizeof(ushort);
return true;
}
if (typeof(T) == typeof(uint))
{
isSigned = false;
size = sizeof(uint);
return true;
}
if (typeof(T) == typeof(nuint))
{
isSigned = false;
size = Unsafe.SizeOf<nuint>();
return true;
}
if (typeof(T) == typeof(ulong))
{
isSigned = false;
size = sizeof(ulong);
return true;
}

if (typeof(T) == typeof(char))
{
isSigned = false;
size = sizeof(char);
return true;
}

if (typeof(T) == typeof(bool))
{
isSigned = false;
size = sizeof(bool);
return true;
}

isSigned = false;
size = default;

return false;
}

private static EnumerableSorter<TElement, TNewKey> CreatePacked<TNewKey, TKey1, TKey2>(
Func<TElement, TKey1> highKeySelector,
Func<TElement, TKey2> lowKeySelector,
int keyPadding,
byte totalSize,
bool key1IsDescending,
bool key2IsDescending,
TNewKey toggleSignBits,
EnumerableSorter<TElement>? next
)
where TNewKey : IBitwiseOperators<TNewKey, TNewKey, TNewKey>, IShiftOperators<TNewKey, int, TNewKey>, IUnsignedNumber<TNewKey>
{
// see github.com/dotnet/runtime/issues/120785 for more information
if (key1IsDescending != key2IsDescending)
{
// make the parent order not apply to the child
TNewKey toggleLowKeyOrder = (TNewKey.One << keyPadding * 8) - TNewKey.One;
toggleSignBits ^= toggleLowKeyOrder;
}

if (toggleSignBits == TNewKey.Zero)
{
//result ^= 0 does nothing
return new EnumerableSorter<TElement, TNewKey>(x =>
{
TKey1 highKey = highKeySelector(x);
TKey2 lowKey = lowKeySelector(x);

TNewKey result = default!;

ref byte resultByteRef = ref Unsafe.As<TNewKey, byte>(ref result);

if (BitConverter.IsLittleEndian)
{
Unsafe.WriteUnaligned(ref resultByteRef, lowKey);

ref byte dest = ref Unsafe.Add(ref resultByteRef, keyPadding);
Unsafe.WriteUnaligned(ref dest, highKey);
}
else
{
Unsafe.WriteUnaligned(ref resultByteRef, highKey);

ref byte dest = ref Unsafe.Add(ref resultByteRef, keyPadding);
Unsafe.WriteUnaligned(ref dest, lowKey);
}

return result;
}, Comparer<TNewKey>.Default, key1IsDescending, next, totalSize);
}

return new EnumerableSorter<TElement, TNewKey>(x =>
{
// highKey = 11111111
TKey1 highKey = highKeySelector(x);
// lowKey = 01111000
TKey2 lowKey = lowKeySelector(x);

// result = 00000000_00000000
TNewKey result = default!;

ref byte resultByteRef = ref Unsafe.As<TNewKey, byte>(ref result);

if (BitConverter.IsLittleEndian)
{
// WriteUnaligned will write the bits in the low end of result
Unsafe.WriteUnaligned(ref resultByteRef, lowKey);
// result = 00000000_01111000
// |--lo--|

// now we want to skip the size of the lowKey writted in the result
ref byte dest = ref Unsafe.Add(ref resultByteRef, keyPadding);
// |--hi--| |--lo--|
// 00000000_01111000
// ^ now we are here

// Write the highKey after lowKey
Unsafe.WriteUnaligned(ref dest, highKey);
// result = 11111111_01111000
// |--hi--| |--lo--|
// toggle the sign bit, to safe convert to unsigned
// (sbyte)0 in binary 00000000 will be 10000000, now (sbyte)0 is in the middle
// basically doing this:
// Middle
// MinValue|----------|----------|MaxValue
// 0
}
else
{
Unsafe.WriteUnaligned(ref resultByteRef, highKey);

ref byte dest = ref Unsafe.Add(ref resultByteRef, keyPadding);
Unsafe.WriteUnaligned(ref dest, lowKey);
}

result ^= toggleSignBits;

return result;
}, Comparer<TNewKey>.Default, key1IsDescending, next, totalSize);
}
}
}
}
Loading
Loading