[libcxx] adds a size-based representation for vector's unstable ABI - #155330
Conversation
cjdb
added
the
libc++
|
@llvm/pr-subscribers-lldb @llvm/pr-subscribers-libcxx Author: Christopher Di Bella (cjdb) Changestl;dr We can significantly improve the runtime performance of This commit changes To use this feature, you'll need to configure libc++ so that it's built with Patch is 113.37 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/155330.diff 6 Files Affected:
diff --git a/libcxx/include/__split_buffer b/libcxx/include/__split_buffer
index 21e58f4abc6b3..04bbfd6c67670 100644
--- a/libcxx/include/__split_buffer
+++ b/libcxx/include/__split_buffer
@@ -13,10 +13,12 @@
#include <__algorithm/max.h>
#include <__algorithm/move.h>
#include <__algorithm/move_backward.h>
+#include <__assert>
#include <__config>
#include <__iterator/distance.h>
#include <__iterator/iterator_traits.h>
#include <__iterator/move_iterator.h>
+#include <__memory/addressof.h>
#include <__memory/allocate_at_least.h>
#include <__memory/allocator.h>
#include <__memory/allocator_traits.h>
@@ -45,13 +47,149 @@ _LIBCPP_PUSH_MACROS
_LIBCPP_BEGIN_NAMESPACE_STD
-// __split_buffer allocates a contiguous chunk of memory and stores objects in the range [__begin_, __end_).
-// It has uninitialized memory in the ranges [__first_, __begin_) and [__end_, __cap_). That allows
-// it to grow both in the front and back without having to move the data.
+template <class _SplitBuffer, class _Tp, class _Allocator>
+class __split_buffer_pointer_layout {
+protected:
+ using value_type = _Tp;
+ using allocator_type = _Allocator;
+ using __alloc_rr _LIBCPP_NODEBUG = __libcpp_remove_reference_t<allocator_type>;
+ using __alloc_traits _LIBCPP_NODEBUG = allocator_traits<__alloc_rr>;
+ using reference = value_type&;
+ using const_reference = const value_type&;
+ using size_type = typename __alloc_traits::size_type;
+ using difference_type = typename __alloc_traits::difference_type;
+ using pointer = typename __alloc_traits::pointer;
+ using const_pointer = typename __alloc_traits::const_pointer;
+ using iterator = pointer;
+ using const_iterator = const_pointer;
-template <class _Tp, class _Allocator = allocator<_Tp> >
-struct __split_buffer {
public:
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __split_buffer_pointer_layout() = default;
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20
+ _LIBCPP_HIDE_FROM_ABI explicit __split_buffer_pointer_layout(const allocator_type& __alloc)
+ : __alloc_(__alloc) {}
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI pointer __data() _NOEXCEPT { return __data_; }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_pointer __data() const _NOEXCEPT { return __data_; }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI pointer begin() _NOEXCEPT { return __begin_; }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_pointer begin() const _NOEXCEPT { return __begin_; }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI pointer end() _NOEXCEPT { return __end_; }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI pointer end() const _NOEXCEPT { return __end_; }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type size() const _NOEXCEPT {
+ return static_cast<size_type>(__end_ - __begin_);
+ }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI bool empty() const _NOEXCEPT { return __begin_ == __end_; }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type capacity() const _NOEXCEPT {
+ return static_cast<size_type>(__cap_ - __data_);
+ }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI allocator_type& __get_allocator() _NOEXCEPT { return __alloc_; }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI allocator_type const& __get_allocator() const _NOEXCEPT {
+ return __alloc_;
+ }
+
+ // Returns the sentinel object directly. Should be used in conjunction with automatic type deduction,
+ // not explicit types.
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI pointer __sentinel() const _NOEXCEPT { return __end_; }
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI pointer __raw_capacity() const _NOEXCEPT { return __cap_; }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __set_data(pointer __new_first) _NOEXCEPT {
+ __data_ = __new_first;
+ }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
+ __set_valid_range(pointer __new_begin, pointer __new_end) _NOEXCEPT {
+ __begin_ = __new_begin;
+ __end_ = __new_end;
+ }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
+ __set_valid_range(pointer __new_begin, size_type __new_size) _NOEXCEPT {
+ __begin_ = __new_begin;
+ __end_ = __begin_ + __new_size;
+ }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __set_sentinel(pointer __new_end) _NOEXCEPT {
+ _LIBCPP_ASSERT_INTERNAL(__data_ <= __new_end, "__new_end cannot precede __data_");
+ __end_ = __new_end;
+ }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __set_sentinel(size_type __new_size) _NOEXCEPT {
+ __end_ = __begin_ + __new_size;
+ }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __set_capacity(size_type __new_capacity) _NOEXCEPT {
+ __cap_ = __data_ + __new_capacity;
+ }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __set_capacity(pointer __new_capacity) _NOEXCEPT {
+ __cap_ = __new_capacity;
+ }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type __front_spare() const _NOEXCEPT {
+ return static_cast<size_type>(__begin_ - __data_);
+ }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type __back_spare() const _NOEXCEPT {
+ return static_cast<size_type>(__cap_ - __end_);
+ }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI reference back() _NOEXCEPT { return *(__end_ - 1); }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reference back() const _NOEXCEPT { return *(__end_ - 1); }
+
+ template <class _OtherLayout>
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __swap_without_allocator(_OtherLayout& __other) _NOEXCEPT {
+ std::swap(__data_, __other.__data_);
+ std::swap(__begin_, __other.__begin_);
+ std::swap(__cap_, __other.__cap_);
+ std::swap(__end_, __other.__end_);
+ }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void swap(__split_buffer_pointer_layout& __other) _NOEXCEPT {
+ __swap_without_allocator(__other);
+ std::__swap_allocator(__alloc_, __other.__alloc_);
+ }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __reset() _NOEXCEPT {
+ __data_ = nullptr;
+ __begin_ = nullptr;
+ __end_ = nullptr;
+ __cap_ = nullptr;
+ }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
+ __copy_without_alloc(__split_buffer_pointer_layout const& __other)
+ _NOEXCEPT_(is_nothrow_copy_assignable<pointer>::value) {
+ __data_ = __other.__data_;
+ __begin_ = __other.__begin_;
+ __cap_ = __other.__cap_;
+ __end_ = __other.__end_;
+ }
+
+private:
+ pointer __data_ = nullptr;
+ pointer __begin_ = nullptr;
+ pointer __end_ = nullptr;
+ _LIBCPP_COMPRESSED_PAIR(pointer, __cap_ = nullptr, allocator_type, __alloc_);
+
+ template <class, class, class>
+ friend class __split_buffer_pointer_layout;
+};
+
+template <class _SplitBuffer, class _Tp, class _Allocator>
+class __split_buffer_size_layout {
+protected:
using value_type = _Tp;
using allocator_type = _Allocator;
using __alloc_rr _LIBCPP_NODEBUG = __libcpp_remove_reference_t<allocator_type>;
@@ -65,6 +203,170 @@ public:
using iterator = pointer;
using const_iterator = const_pointer;
+public:
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __split_buffer_size_layout() = default;
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI explicit __split_buffer_size_layout(const allocator_type& __alloc)
+ : __alloc_(__alloc) {}
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI pointer __data() _NOEXCEPT { return __data_; }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_pointer __data() const _NOEXCEPT { return __data_; }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI pointer begin() _NOEXCEPT { return __begin_; }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_pointer begin() const _NOEXCEPT { return __begin_; }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI pointer end() _NOEXCEPT { return __begin_ + __size_; }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI pointer end() const _NOEXCEPT { return __begin_ + __size_; }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type size() const _NOEXCEPT { return __size_; }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI bool empty() const _NOEXCEPT { return __size_ == 0; }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type capacity() const _NOEXCEPT { return __cap_; }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI allocator_type& __get_allocator() _NOEXCEPT { return __alloc_; }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI allocator_type const& __get_allocator() const _NOEXCEPT {
+ return __alloc_;
+ }
+
+ // Returns the sentinel object directly. Should be used in conjunction with automatic type deduction,
+ // not explicit types.
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type __sentinel() const _NOEXCEPT { return __size_; }
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type __raw_capacity() const _NOEXCEPT { return __cap_; }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __set_data(pointer __new_first) _NOEXCEPT {
+ __data_ = __new_first;
+ }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
+ __set_valid_range(pointer __new_begin, pointer __new_end) _NOEXCEPT {
+ // Size-based __split_buffers track their size directly: we need to explicitly update the size
+ // when the front is adjusted.
+ __size_ -= __new_begin - __begin_;
+ __begin_ = __new_begin;
+ __set_sentinel(__new_end);
+ }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
+ __set_valid_range(pointer __new_begin, size_type __new_size) _NOEXCEPT {
+ // Size-based __split_buffers track their size directly: we need to explicitly update the size
+ // when the front is adjusted.
+ __size_ -= __new_begin - __begin_;
+ __begin_ = __new_begin;
+ __set_sentinel(__new_size);
+ }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __set_sentinel(pointer __new_end) _NOEXCEPT {
+ _LIBCPP_ASSERT_INTERNAL(__data_ <= __new_end, "__new_end cannot precede __data_");
+ __size_ += __new_end - end();
+ }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __set_sentinel(size_type __new_size) _NOEXCEPT {
+ __size_ = __new_size;
+ }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __set_capacity(size_type __new_capacity) _NOEXCEPT {
+ __cap_ = __new_capacity;
+ }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __set_capacity(pointer __new_capacity) _NOEXCEPT {
+ __cap_ = __new_capacity - __begin_;
+ }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type __front_spare() const _NOEXCEPT {
+ return static_cast<size_type>(__begin_ - __data_);
+ }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type __back_spare() const _NOEXCEPT {
+ // `__cap_ - __end_` tells us the total number of spares when in size-mode. We need to remove
+ // the __front_spare from the count.
+ return __cap_ - __size_ - __front_spare();
+ }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI reference back() _NOEXCEPT { return __begin_[__size_ - 1]; }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reference back() const _NOEXCEPT {
+ return __begin_[__size_ - 1];
+ }
+
+ template <class _Data2>
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __swap_without_allocator(_Data2& __other) _NOEXCEPT {
+ std::swap(__data_, __other.__data_);
+ std::swap(__begin_, __other.__begin_);
+ std::swap(__cap_, __other.__cap_);
+ std::swap(__size_, __other.__size_);
+ }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void swap(__split_buffer_size_layout& __other) _NOEXCEPT {
+ __swap_without_allocator(__other);
+ std::__swap_allocator(__alloc_, __other.__alloc_);
+ }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __reset() _NOEXCEPT {
+ __data_ = nullptr;
+ __begin_ = nullptr;
+ __size_ = 0;
+ __cap_ = 0;
+ }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
+ __copy_without_alloc(__split_buffer_size_layout const& __other)
+ _NOEXCEPT_(is_nothrow_copy_assignable<pointer>::value) {
+ __data_ = __other.__data_;
+ __begin_ = __other.__begin_;
+ __cap_ = __other.__cap_;
+ __size_ = __other.__size_;
+ }
+
+private:
+ pointer __data_ = nullptr;
+ pointer __begin_ = nullptr;
+ size_type __size_ = 0;
+ size_type __cap_ = 0;
+ _LIBCPP_NO_UNIQUE_ADDRESS allocator_type __alloc_;
+
+ template <class, class, class>
+ friend class __split_buffer_size_layout;
+};
+
+// __split_buffer allocates a contiguous chunk of memory and stores objects in the range [__begin_, __end_).
+// It has uninitialized memory in the ranges [__data_, __begin_) and [__end_, __cap_). That allows
+// it to grow both in the front and back without having to move the data.
+
+template <class _Tp,
+ class _Allocator = allocator<_Tp>,
+ template <class, class, class> class _Layout = __split_buffer_pointer_layout>
+struct __split_buffer : _Layout<__split_buffer<_Tp, _Allocator, _Layout>, _Tp, _Allocator> {
+ using __base_type _LIBCPP_NODEBUG = _Layout<__split_buffer<_Tp, _Allocator, _Layout>, _Tp, _Allocator>;
+ using __base_type::__back_spare;
+ using __base_type::__copy_without_alloc;
+ using __base_type::__data;
+ using __base_type::__front_spare;
+ using __base_type::__get_allocator;
+ using __base_type::__reset;
+ using __base_type::__set_capacity;
+ using __base_type::__set_data;
+ using __base_type::__set_sentinel;
+ using __base_type::__set_valid_range;
+ using __base_type::__swap_without_allocator;
+
+ using typename __base_type::__alloc_rr;
+ using typename __base_type::__alloc_traits;
+ using typename __base_type::allocator_type;
+ using typename __base_type::const_iterator;
+ using typename __base_type::const_pointer;
+ using typename __base_type::const_reference;
+ using typename __base_type::difference_type;
+ using typename __base_type::iterator;
+ using typename __base_type::pointer;
+ using typename __base_type::reference;
+ using typename __base_type::size_type;
+ using typename __base_type::value_type;
+
// A __split_buffer contains the following members which may be trivially relocatable:
// - pointer: may be trivially relocatable, so it's checked
// - allocator_type: may be trivially relocatable, so it's checked
@@ -78,23 +380,15 @@ public:
__split_buffer,
void>;
- pointer __first_;
- pointer __begin_;
- pointer __end_;
- _LIBCPP_COMPRESSED_PAIR(pointer, __cap_, allocator_type, __alloc_);
-
__split_buffer(const __split_buffer&) = delete;
__split_buffer& operator=(const __split_buffer&) = delete;
- _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __split_buffer()
- _NOEXCEPT_(is_nothrow_default_constructible<allocator_type>::value)
- : __first_(nullptr), __begin_(nullptr), __end_(nullptr), __cap_(nullptr) {}
+ _LIBCPP_HIDE_FROM_ABI __split_buffer() = default;
- _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI explicit __split_buffer(__alloc_rr& __a)
- : __first_(nullptr), __begin_(nullptr), __end_(nullptr), __cap_(nullptr), __alloc_(__a) {}
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI explicit __split_buffer(__alloc_rr& __a) : __base_type(__a) {}
_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI explicit __split_buffer(const __alloc_rr& __a)
- : __first_(nullptr), __begin_(nullptr), __end_(nullptr), __cap_(nullptr), __alloc_(__a) {}
+ : __base_type(__a) {}
_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI
__split_buffer(size_type __cap, size_type __start, __alloc_rr& __a);
@@ -111,36 +405,16 @@ public:
_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI ~__split_buffer();
- _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator begin() _NOEXCEPT { return __begin_; }
- _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_iterator begin() const _NOEXCEPT { return __begin_; }
-
- _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator end() _NOEXCEPT { return __end_; }
- _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_iterator end() const _NOEXCEPT { return __end_; }
-
- _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void clear() _NOEXCEPT { __destruct_at_end(__begin_); }
-
- _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type size() const {
- return static_cast<size_type>(__end_ - __begin_);
- }
+ using __base_type::back;
+ using __base_type::begin;
+ using __base_type::capacity;
+ using __base_type::empty;
+ using __base_type::end;
+ using __base_type::size;
- _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI bool empty() const { return __end_ == __begin_; }
-
- _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type capacity() const {
- return static_cast<size_type>(__cap_ - __first_);
- }
-
- _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type __front_spare() const {
- return static_cast<size_type>(__begin_ - __first_);
- }
-
- _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type __back_spare() const {
- return static_cast<size_type>(__cap_ - __end_);
- }
-
- _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI reference front() { return *__begin_; }
- _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reference front() const { return *__begin_; }
- _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI reference back() { return *(__end_ - 1); }
- _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reference back() const { return *(__end_ - 1); }
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void clear() _NOEXCEPT { __destruct_at_end(begin()); }
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI reference front() { return *begin(); }
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reference front() const { return *begin(); }
_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void shrink_to_fit() _NOEXCEPT;
@@ -149,8 +423,8 @@ public:
template <class... _Args>
_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void emplace_back(_Args&&... __args);
- _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void pop_front() { __destruct_at_begin(__begin_ + 1); }
- _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void pop_back() { __destruct_at_end(__end_ - 1); }
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void pop_front() { __destruct_at_begin(begin() + 1); }
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void pop_back() { __destruct_at_end(end() - 1); }
_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __construct_at_end(size_type __n);
_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __construct_at_end(size_type __n, const_reference __x);
@@ -184,242 +458,232 @@ public:
_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void swap(__split_buffer& __x)
_NOEXCEPT_(!__alloc_traits::propagate_on_container_swap::value || __is_nothrow_swappable_v<__alloc_rr>);
- _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI bool __invariants() const;
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI bool __invariants() const {
+ if (__data() == nullptr) {
+ if (begin() != nullptr)
+ return false;
+
+ if (!empty())
+ return false;
+
+ if (capacity() != 0)
+ return false;
+
+ return true;
+ } else {
+ if (begin() < __data())
+ return false;
+
+ if (capacity() < size())
+ return false;
+
+ if (end() < begin())
+ return false;
+
+ return true;
+ }
+ }
private:
_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __move_assign_alloc(__split_buffer& __c, true_type)
_NOEXCEPT_(is_nothrow_move_assignable<allocator_type>::value) {
- __alloc_ = std::move(__c.__alloc_);
+ __get_allocator() = std::move(__c.__get_allocator());
}
_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __move_assign...
[truncated]
|
877d560 to
ccc8d09
Compare
|
✅ With the latest revision this PR passed the C/C++ code formatter. |
|
The LLDB and GDB pretty-printers will require some tweaking again, but those changes haven't been applied yet. |
|
After having attempted to build all of our code with this change, I've observed a few things. None of them are really surprising, but putting some countermeasures in place to nudge users in the right direction would be worthwhile.
|
ccc8d09 to
469c68c
Compare
|
Ping |
d2b9ec3 to
1e7c6fe
Compare
|
@ldionne Would you mind reviewing this, please? |
c15c6d7 to
f314efd
Compare
|
@jyknight @rnk, the libc++ maintainers have said that they're in favour of the overall direction of size-based vector, but it looks like they don't have the bandwidth to review this PR. Would you mind assisting with the review process so that we can unblock the merging of this patch, please? Other |
f314efd to
c90af22
Compare
**tl;dr** We can significantly improve the runtime performance of `std::vector` by changing its representation from three pointers to one pointer and two integers. This document explains the details of this change, along with the justifications for making it. See the [RFC] for more information. This commit changes `std::vector`'s representation in a similar manner to `__split_buffer` in llvm#139632. We introduce a layout type that tracks implementation details that differ between the pointer-based vector and size-based vector representations. `vector` does not parameterise its layout type, unlike `__split_buffer`. `__split_buffer` is used by both `vector` and `deque`, and being able to take an ABI break on `vector` doesn't automatically mean that users can also tolerate a break on `deque`. The most convenient way to work around this problem is to parameterise the layout type. Users of `std::vector` should not depend on its layout, and libc++ has no internal reason to toggle it, so we keep the representation concrete. To use this feature, you'll need to configure libc++ so that it's built with `-DLIBCXX_UNSTABLE_ABI=Yes`. [RFC]: https://discourse.llvm.org/t/adding-a-size-based-vector-to-libc-s-unstable-abi/86306
c90af22 to
ba94eeb
Compare
🐧 Linux x64 Test Results
✅ The build succeeded and all tests passed. |
…uts (#202438) PR #155330 changes `std::vector` from unconditionally using three pointers to represent its layout to potentially using three pointers or a begin pointer and two integers. This commit changes LLDB so that it can robustly work with the legacy vector layout, the new pointer layout, and the new size-based layout.
|
PR is passing all CI tests. @ldionne this should be ready to merge with your LGTM! |
|
|
||
| _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __reset_without_allocator() _NOEXCEPT; | ||
| _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __swap(__vector_layout& __other) _NOEXCEPT; | ||
| _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __swap_layouts(_SplitBuffer& __other) _NOEXCEPT; |
There was a problem hiding this comment.
I think this is dead code (and also the two definitions).
|
|
||
| template <class... _Args> | ||
| _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI | ||
| _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __emplace_back_result_t emplace_back(_Args&&... __args) { |
There was a problem hiding this comment.
This function was previously written using __if_likely_else on purpose (to workaround something really subtle with Clang), see the discussion in #94379. Unless the underlying Clang issue has been fixed, this refactoring would unfortunately introduce a codegen regression. Since this is a very important function for vector (arguably the most important one?), I think we should go back to using it.
Now, regarding whether we can avoid setting __end = __emplace_back_slow_path(std::forward<_Args>(__args)...); from this function, I think we'd want to check the codegen before and after. IIRC, the idea was to make the flow of data explicitly visible to the compiler because it allowed it to generate better code (that's why we didn't call back() again in the pre-refactor version).
We're on the same page that the new code is better in the sense of readability and reuse, but we had decided to make a conscious readability tradeoff since this function is very performance sensitive.
Now, if we find out that we can avoid the _end = __emplace_back_slow_path(...) without hurting code generation (I'd be surprised because something would have had to change), that means we can avoid bifurcating __emplace_back. Otherwise, I'd bifurcate it for the size/pointer based vector layouts and accept it as tech debt.
There was a problem hiding this comment.
Let's defer this change to a later PR (that I'll open ~soon). I had some fairly promising codegen experiments with removing __if_likely_else before pushing the commit for review. I've been trying to get some macrobenchmarks since you've flagged, but it looks like there's a few blockers for a timely turnaround on that, and I don't want to block on something that can be done independently.
There was a problem hiding this comment.
Now, regarding whether we can avoid setting
__end = __emplace_back_slow_path(std::forward<_Args>(__args)...);from this function, I think we'd want to check the codegen before and after. IIRC, the idea was to make the flow of data explicitly visible to the compiler because it allowed it to generate better code (that's why we didn't callback()again in the pre-refactor version).We're on the same page that the new code is better in the sense of readability and reuse, but we had decided to make a conscious readability tradeoff since this function is very performance sensitive.
Now, if we find out that we can avoid the
_end = __emplace_back_slow_path(...)without hurting code generation (I'd be surprised because something would have had to change), that means we can avoid bifurcating__emplace_back. Otherwise, I'd bifurcate it for the size/pointer based vector layouts and accept it as tech debt.
I modified the above experiment to look at what happens when I toggle this change (you'll want to look for IMPLICIT_SET_END). I'm by no means an expert in assembly, but it looks like we get simpler codegen in both implementations across all experiments to me, when we stop assigning __end in emplace_back?
There was a problem hiding this comment.
I've rolled this back for the time being, and will make a follow-up PR once the benchmark results have come through.
There was a problem hiding this comment.
Thanks. The original change that introduced this pattern explained the reason behind it: 6fe4e03
Doing my own test similar to yours, I also don't see an immediate problem with dropping the explicit store. But I'd rather do it intentionally in a dedicated PR where we can fully investigate.
|
I'm excited to see this is coming along! |
ldionne
left a comment
ldionne
left a comment
There was a problem hiding this comment.
This has been a long time coming, but I think this is good to go. Thanks for the diligence and patience on this review, this was non-trivial.
Since this touches vector in non-trivial ways, I do expect we might need some follow-ups on this. I don't know exactly what, but I wouldn't be surprised if something came up, since it often does when we touch something as ubiquitous (e.g. performance change, debug size impact, or else). Let's just keep an eye out for this, and when/if this happens, we'll decide whether to temporarily roll-back or fix forward.
Thanks!
|
Thank you! I'm really happy to see this one land. (Ignore the draft PR conversions: that was meant to be applied to a different PR.) |
As I mentioned offline: I'm much more responsive to DMs than to email notifications. I'll be much more likely to engage if I'm pinged on Discord, in addition to any tags in this PR or follow-up PRs made by others. |
6 participants
tl;dr We can significantly improve the runtime performance of
std::vectorby changing its representation from three pointers to one pointer and two integers. This document explains the details of this change, along with the justifications for making it. See the RFC for more information.This commit changes
std::vector's representation in a similar manner to__split_bufferin #139632. We introduce a layout type that tracks implementation details that differ between the pointer-based vector and size-based vector representations.vectordoes not parameterise its layout type, unlike__split_buffer.__split_bufferis used by bothvectoranddeque, and being able to take an ABI break onvectordoesn't automatically mean that users can also tolerate a break ondeque. The most convenient way to work around this problem is to parameterise the layout type. Users ofstd::vectorshould not depend on its layout, and libc++ has no internal reason to toggle it, so we keep the representation concrete.To use this feature, you'll need to configure libc++ so that it's built with
-DLIBCXX_UNSTABLE_ABI=Yes.