Skip to content

Add algorithm-specific detection macros for vectorization #5801

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 33 commits into from
Oct 29, 2025
Merged

Add algorithm-specific detection macros for vectorization #5801

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
33 commits
Select commit Hold shift + click to select a range
29f7aa8
Set up the new macro system.
StephanTLavavej Oct 24, 2025
9df2f9b
Inspect `_VECTORIZED_FOR_X64_X86`.
StephanTLavavej Oct 24, 2025
4efcf9d
Improve comments for vectorization type traits.
StephanTLavavej Oct 24, 2025
11a385e
_VECTORIZED_MISMATCH. (note _Lex_compare_memcmp_classify_elements)
StephanTLavavej Oct 24, 2025
9c4490b
_VECTORIZED_IS_SORTED_UNTIL.
StephanTLavavej Oct 24, 2025
44afd8c
_VECTORIZED_ROTATE.
StephanTLavavej Oct 24, 2025
d58cc6c
_VECTORIZED_SWAP_RANGES.
StephanTLavavej Oct 24, 2025
f79f444
_VECTORIZED_REVERSE.
StephanTLavavej Oct 24, 2025
6817749
_VECTORIZED_REMOVE.
StephanTLavavej Oct 24, 2025
ad78933
_VECTORIZED_INCLUDES.
StephanTLavavej Oct 24, 2025
547fd56
_VECTORIZED_MINMAX.
StephanTLavavej Oct 24, 2025
cbc0361
_VECTORIZED_MINMAX_ELEMENT. (note interaction with _VECTORIZED_MINMAX)
StephanTLavavej Oct 24, 2025
465b4c5
_VECTORIZED_COUNT.
StephanTLavavej Oct 24, 2025
a21bbae
_VECTORIZED_SEARCH.
StephanTLavavej Oct 24, 2025
649a2fd
_VECTORIZED_ADJACENT_FIND.
StephanTLavavej Oct 24, 2025
6747f12
_VECTORIZED_FIND_FIRST_OF.
StephanTLavavej Oct 24, 2025
d410936
_VECTORIZED_FIND_LAST_OF.
StephanTLavavej Oct 24, 2025
b2485ab
_VECTORIZED_FIND.
StephanTLavavej Oct 24, 2025
071af15
_VECTORIZED_SEARCH_N.
StephanTLavavej Oct 24, 2025
94f5ea6
_VECTORIZED_REMOVE_COPY.
StephanTLavavej Oct 24, 2025
d81cf8f
_VECTORIZED_FIND_LAST.
StephanTLavavej Oct 24, 2025
c2a314d
_VECTORIZED_REVERSE_COPY.
StephanTLavavej Oct 24, 2025
2e58939
_VECTORIZED_REPLACE.
StephanTLavavej Oct 24, 2025
86f8a90
_VECTORIZED_FIND_END.
StephanTLavavej Oct 24, 2025
13f8b50
_VECTORIZED_FIND_LAST_NOT_OF.
StephanTLavavej Oct 24, 2025
cbf48ae
_VECTORIZED_UNIQUE.
StephanTLavavej Oct 24, 2025
5af3ecc
_VECTORIZED_UNIQUE_COPY.
StephanTLavavej Oct 24, 2025
741b5b9
_VECTORIZED_BITSET_TO_STRING.
StephanTLavavej Oct 24, 2025
a19efcd
_VECTORIZED_BITSET_FROM_STRING.
StephanTLavavej Oct 24, 2025
834eea4
Remove/shrink _VECTORIZED_FOR_X64_X86 regions.
StephanTLavavej Oct 24, 2025
a8b6015
Guard the helper traits to finish removing _VECTORIZED_FOR_X64_X86.
StephanTLavavej Oct 24, 2025
391b33e
Define the algorithm macros.
StephanTLavavej Oct 24, 2025
c546a48
Unify _VECTORIZED_FIND_LAST_NOT_OF into _VECTORIZED_FIND_LAST.
StephanTLavavej Oct 25, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 24 additions & 24 deletions stl/inc/__msvc_string_view.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ _STL_DISABLE_CLANG_WARNINGS
#pragma push_macro("new")
#undef new

#if _USE_STD_VECTOR_ALGORITHMS
#if _VECTORIZED_FOR_X64_X86
extern "C" {
// The "noalias" attribute tells the compiler optimizer that pointers going into these hand-vectorized algorithms
// won't be stored beyond the lifetime of the function, and that the function will only reference arrays denoted by
Expand Down Expand Up @@ -158,7 +158,7 @@ size_t _Find_last_not_of_pos_vectorized(const _Ty1* const _Haystack, const size_

_STD_END

#endif // _USE_STD_VECTOR_ALGORITHMS
#endif // ^^^ _VECTORIZED_FOR_X64_X86 ^^^

_STD_BEGIN
#ifdef __clang__
Expand Down Expand Up @@ -360,7 +360,7 @@ struct _WChar_traits : private _Char_traits<_Elem, unsigned short> {
_NODISCARD static _CONSTEXPR17 int compare(_In_reads_(_Count) const _Elem* const _First1,
_In_reads_(_Count) const _Elem* const _First2, const size_t _Count) noexcept /* strengthened */ {
// compare [_First1, _First1 + _Count) with [_First2, ...)
#if _USE_STD_VECTOR_ALGORITHMS
#if _VECTORIZED_FOR_X64_X86
if (!_STD _Is_constant_evaluated()) {
// TRANSITION, GH-2289: Use vectorized algorithms for better performance than __builtin_wmemcmp.
const size_t _Pos = _Mismatch_vectorized<sizeof(_Elem)>(_First1, _First2, _Count);
Expand All @@ -370,7 +370,7 @@ struct _WChar_traits : private _Char_traits<_Elem, unsigned short> {
return _First1[_Pos] < _First2[_Pos] ? -1 : +1;
}
}
#endif // ^^^ _USE_STD_VECTOR_ALGORITHMS ^^^
#endif // ^^^ _VECTORIZED_FOR_X64_X86 ^^^

if constexpr (is_same_v<_Elem, wchar_t>) {
return __builtin_wmemcmp(_First1, _First2, _Count);
Expand Down Expand Up @@ -728,7 +728,7 @@ constexpr size_t _Traits_find(_In_reads_(_Hay_size) const _Traits_ptr_t<_Traits>
return _Start_at;
}

#if _USE_STD_VECTOR_ALGORITHMS
#if _VECTORIZED_FOR_X64_X86
if constexpr (_Is_implementation_handled_char_traits<_Traits>) {
if (!_STD _Is_constant_evaluated()) {
const auto _End = _Haystack + _Hay_size;
Expand All @@ -741,7 +741,7 @@ constexpr size_t _Traits_find(_In_reads_(_Hay_size) const _Traits_ptr_t<_Traits>
}
}
}
#endif // _USE_STD_VECTOR_ALGORITHMS
#endif // ^^^ _VECTORIZED_FOR_X64_X86 ^^^

const auto _Possible_matches_end = _Haystack + (_Hay_size - _Needle_size) + 1;
for (auto _Match_try = _Haystack + _Start_at;; ++_Match_try) {
Expand All @@ -764,7 +764,7 @@ constexpr size_t _Traits_find_ch(_In_reads_(_Hay_size) const _Traits_ptr_t<_Trai
return static_cast<size_t>(-1); // (npos) no room for match
}

#if _USE_STD_VECTOR_ALGORITHMS
#if _VECTORIZED_FOR_X64_X86
if constexpr (_Is_implementation_handled_char_traits<_Traits>) {
if (!_STD _Is_constant_evaluated()) {
const auto _End = _Haystack + _Hay_size;
Expand All @@ -777,7 +777,7 @@ constexpr size_t _Traits_find_ch(_In_reads_(_Hay_size) const _Traits_ptr_t<_Trai
}
}
}
#endif // _USE_STD_VECTOR_ALGORITHMS
#endif // ^^^ _VECTORIZED_FOR_X64_X86 ^^^

const auto _Found_at = _Traits::find(_Haystack + _Start_at, _Hay_size - _Start_at, _Ch);
if (_Found_at) {
Expand All @@ -802,7 +802,7 @@ constexpr size_t _Traits_rfind(_In_reads_(_Hay_size) const _Traits_ptr_t<_Traits

const size_t _Actual_start_at = (_STD min) (_Start_at, _Hay_size - _Needle_size);

#if _USE_STD_VECTOR_ALGORITHMS
#if _VECTORIZED_FOR_X64_X86
if constexpr (_Is_implementation_handled_char_traits<_Traits>) {
if (!_STD _Is_constant_evaluated()) {
// _Find_end_vectorized takes into account the needle length when locating the search start.
Expand All @@ -819,7 +819,7 @@ constexpr size_t _Traits_rfind(_In_reads_(_Hay_size) const _Traits_ptr_t<_Traits
}
}
}
#endif // _USE_STD_VECTOR_ALGORITHMS
#endif // ^^^ _VECTORIZED_FOR_X64_X86 ^^^

for (auto _Match_try = _Haystack + _Actual_start_at;; --_Match_try) {
if (_Traits::eq(*_Match_try, *_Needle) && _Traits::compare(_Match_try, _Needle, _Needle_size) == 0) {
Expand All @@ -843,7 +843,7 @@ constexpr size_t _Traits_rfind_ch(_In_reads_(_Hay_size) const _Traits_ptr_t<_Tra

const size_t _Actual_start_at = (_STD min) (_Start_at, _Hay_size - 1);

#if _USE_STD_VECTOR_ALGORITHMS
#if _VECTORIZED_FOR_X64_X86
if constexpr (_Is_implementation_handled_char_traits<_Traits>) {
if (!_STD _Is_constant_evaluated()) {
const auto _End = _Haystack + _Actual_start_at + 1;
Expand All @@ -856,7 +856,7 @@ constexpr size_t _Traits_rfind_ch(_In_reads_(_Hay_size) const _Traits_ptr_t<_Tra
}
}
}
#endif // _USE_STD_VECTOR_ALGORITHMS
#endif // ^^^ _VECTORIZED_FOR_X64_X86 ^^^

for (auto _Match_try = _Haystack + _Actual_start_at;; --_Match_try) {
if (_Traits::eq(*_Match_try, _Ch)) {
Expand Down Expand Up @@ -934,7 +934,7 @@ constexpr size_t _Traits_find_first_of(_In_reads_(_Hay_size) const _Traits_ptr_t
const auto _Hay_end = _Haystack + _Hay_size;

if constexpr (_Is_implementation_handled_char_traits<_Traits>) {
#if _USE_STD_VECTOR_ALGORITHMS
#if _VECTORIZED_FOR_X64_X86
if (!_STD _Is_constant_evaluated()) {
const size_t _Remaining_size = _Hay_size - _Start_at;
if (_Remaining_size + _Needle_size >= _Threshold_find_first_of) {
Expand All @@ -945,7 +945,7 @@ constexpr size_t _Traits_find_first_of(_In_reads_(_Hay_size) const _Traits_ptr_t
return _Pos;
}
}
#endif // _USE_STD_VECTOR_ALGORITHMS
#endif // ^^^ _VECTORIZED_FOR_X64_X86 ^^^

_String_bitmap<typename _Traits::char_type> _Matches;

Expand Down Expand Up @@ -983,7 +983,7 @@ constexpr size_t _Traits_find_last_of(_In_reads_(_Hay_size) const _Traits_ptr_t<

if constexpr (_Is_implementation_handled_char_traits<_Traits>) {
using _Elem = typename _Traits::char_type;
#if _USE_STD_VECTOR_ALGORITHMS
#if _VECTORIZED_FOR_X64_X86
if constexpr (sizeof(_Elem) <= 2) {
if (!_STD _Is_constant_evaluated()) {
const size_t _Remaining_size = _Hay_start + 1;
Expand All @@ -992,7 +992,7 @@ constexpr size_t _Traits_find_last_of(_In_reads_(_Hay_size) const _Traits_ptr_t<
}
}
}
#endif // _USE_STD_VECTOR_ALGORITHMS
#endif // ^^^ _VECTORIZED_FOR_X64_X86 ^^^

_String_bitmap<_Elem> _Matches;
if (_Matches._Mark(_Needle, _Needle + _Needle_size)) {
Expand Down Expand Up @@ -1035,7 +1035,7 @@ constexpr size_t _Traits_find_first_not_of(_In_reads_(_Hay_size) const _Traits_p

if constexpr (_Is_implementation_handled_char_traits<_Traits>) {
using _Elem = typename _Traits::char_type;
#if _USE_STD_VECTOR_ALGORITHMS
#if _VECTORIZED_FOR_X64_X86
if constexpr (sizeof(_Elem) <= 2) {
if (!_STD _Is_constant_evaluated()) {
const size_t _Remaining_size = _Hay_size - _Start_at;
Expand All @@ -1048,7 +1048,7 @@ constexpr size_t _Traits_find_first_not_of(_In_reads_(_Hay_size) const _Traits_p
}
}
}
#endif // _USE_STD_VECTOR_ALGORITHMS
#endif // ^^^ _VECTORIZED_FOR_X64_X86 ^^^

_String_bitmap<_Elem> _Matches;
if (_Matches._Mark(_Needle, _Needle + _Needle_size)) {
Expand Down Expand Up @@ -1082,7 +1082,7 @@ constexpr size_t _Traits_find_not_ch(_In_reads_(_Hay_size) const _Traits_ptr_t<_

const auto _End = _Haystack + _Hay_size;

#if _USE_STD_VECTOR_ALGORITHMS
#if _VECTORIZED_FOR_X64_X86
if constexpr (_Is_implementation_handled_char_traits<_Traits>) {
if (!_STD _Is_constant_evaluated()) {
const auto _Result = _STD _Find_not_ch_vectorized(_Haystack + _Start_at, _End, _Ch);
Expand All @@ -1093,7 +1093,7 @@ constexpr size_t _Traits_find_not_ch(_In_reads_(_Hay_size) const _Traits_ptr_t<_
}
}
}
#endif // _USE_STD_VECTOR_ALGORITHMS
#endif // ^^^ _VECTORIZED_FOR_X64_X86 ^^^

for (auto _Match_try = _Haystack + _Start_at; _Match_try < _End; ++_Match_try) {
if (!_Traits::eq(*_Match_try, _Ch)) {
Expand All @@ -1117,7 +1117,7 @@ constexpr size_t _Traits_find_last_not_of(_In_reads_(_Hay_size) const _Traits_pt

if constexpr (_Is_implementation_handled_char_traits<_Traits>) {
using _Elem = typename _Traits::char_type;
#if _USE_STD_VECTOR_ALGORITHMS
#if _VECTORIZED_FOR_X64_X86
if constexpr (sizeof(_Elem) <= 2) {
if (!_STD _Is_constant_evaluated()) {
const size_t _Remaining_size = _Hay_start + 1;
Expand All @@ -1126,7 +1126,7 @@ constexpr size_t _Traits_find_last_not_of(_In_reads_(_Hay_size) const _Traits_pt
}
}
}
#endif // _USE_STD_VECTOR_ALGORITHMS
#endif // ^^^ _VECTORIZED_FOR_X64_X86 ^^^

_String_bitmap<_Elem> _Matches;
if (_Matches._Mark(_Needle, _Needle + _Needle_size)) {
Expand Down Expand Up @@ -1165,13 +1165,13 @@ constexpr size_t _Traits_rfind_not_ch(_In_reads_(_Hay_size) const _Traits_ptr_t<

const size_t _Actual_start_at = (_STD min) (_Start_at, _Hay_size - 1);

#if _USE_STD_VECTOR_ALGORITHMS
#if _VECTORIZED_FOR_X64_X86
if constexpr (_Is_implementation_handled_char_traits<_Traits>) {
if (!_STD _Is_constant_evaluated()) {
return _STD _Find_last_not_ch_pos_vectorized(_Haystack, _Haystack + _Actual_start_at + 1, _Ch);
}
}
#endif // _USE_STD_VECTOR_ALGORITHMS
#endif // ^^^ _VECTORIZED_FOR_X64_X86 ^^^

for (auto _Match_try = _Haystack + _Actual_start_at;; --_Match_try) {
if (!_Traits::eq(*_Match_try, _Ch)) {
Expand Down
Loading