diff --git a/libcxx/docs/ABIGuarantees.rst b/libcxx/docs/ABIGuarantees.rst index c566118542210..c5ebe9ad76be8 100644 --- a/libcxx/docs/ABIGuarantees.rst +++ b/libcxx/docs/ABIGuarantees.rst @@ -58,6 +58,56 @@ most cases. This significantly restructures how ``function`` is written to provide better performance, but is currently not ABI stable. +``_LIBCPP_ABI_VECTOR_LAYOUT_SIZE_BASED`` +---------------------------------------- +Changes the layout of :cpp:type:`std::vector` from pointer-based to size-based. + +libc++ supports two different data layouts for :cpp:type:`std::vector`: + +.. list-table:: + :header-rows: 1 + + * - **Layout** + - ABI + - Description + * - Pointer-based layout + - Stable ABI (default) + - :cpp:type:`std::vector` uses three pointers to manage its state: + + * A pointer to the beginning of the buffer (:cpp:expr:`begin_`); + * A pointer to where the next element should be inserted (:cpp:expr:`end_`); and + * A pointer to the end of the buffer (:cpp:expr:`cap_`). + + This layout causes :cpp:type:`vector`'s implementation details to be pointer-oriented. + The following methods are of particular interest: + + * :cpp:expr:`vector::size()` returns :cpp:expr:`end_ - begin_`; + * :cpp:expr:`vector::capacity()` returns :cpp:expr:`cap_ - begin_`; and + * :cpp:expr:`vector::end()` returns :cpp:expr:`end_`. + + This is the original layout for libc++'s :cpp:type:`std::vector` implementation, and + is the default layout as a result. + + * - Size-based layout + - Unstable ABI (opt-in) + - :cpp:type:`std::vector` uses a pointer and two integers to manage its state: + + * A pointer to the beginning of the buffer (:cpp:expr:`begin_`); + * An integer storing how many elements are in the vector (:cpp:expr:`size_`); and + * An integer storing how many elements the vector can potentially hold before needing + to reallocate (:cpp:expr:`capacity_`). + + This layout causes :cpp:type:`vector`'s implementation details to be integer-oriented. + The following methods are of particular interest: + + * :cpp:expr:`vector::size()` returns :cpp:expr:`size_`; + * :cpp:expr:`vector::capacity()` returns :cpp:expr:`cap_`; and + * :cpp:expr:`vector::end()` returns :cpp:expr:`begin_ + size_`. + + This layout is opt-in, and is incompatible with the pointer-based layout. It has the + potential for significant performance improvements, especially when combined with + :ref:`hardening`. + ``_LIBCPP_ABI_NO_RANDOM_DEVICE_COMPATIBILITY_LAYOUT`` ----------------------------------------------------- This changes the layout of ``random_device`` to only holds state with an implementation that gets entropy from a file diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt index cec9ba06a7a6d..d65e66e221766 100644 --- a/libcxx/include/CMakeLists.txt +++ b/libcxx/include/CMakeLists.txt @@ -962,6 +962,7 @@ set(files __vector/comparison.h __vector/container_traits.h __vector/erase.h + __vector/layout.h __vector/pmr.h __vector/swap.h __vector/vector.h diff --git a/libcxx/include/__configuration/abi.h b/libcxx/include/__configuration/abi.h index 51c82d99eec30..9231271d7e692 100644 --- a/libcxx/include/__configuration/abi.h +++ b/libcxx/include/__configuration/abi.h @@ -71,6 +71,7 @@ # define _LIBCPP_ABI_NO_REVERSE_ITERATOR_SECOND_MEMBER # define _LIBCPP_ABI_OPTIMIZED_FUNCTION # define _LIBCPP_ABI_REGEX_CONSTANTS_NONZERO +# define _LIBCPP_ABI_VECTOR_LAYOUT_SIZE_BASED # define _LIBCPP_ABI_STRING_OPTIMIZED_EXTERNAL_INSTANTIATION # define _LIBCPP_ABI_USE_WRAP_ITER_IN_STD_ARRAY # define _LIBCPP_ABI_USE_WRAP_ITER_IN_STD_STRING_VIEW diff --git a/libcxx/include/__split_buffer b/libcxx/include/__split_buffer index 27ec8b7f988d3..1d50486947f08 100644 --- a/libcxx/include/__split_buffer +++ b/libcxx/include/__split_buffer @@ -25,6 +25,7 @@ #include <__memory/compressed_pair.h> #include <__memory/pointer_traits.h> #include <__memory/swap_allocator.h> +#include <__memory/uninitialized_algorithms.h> #include <__type_traits/conditional.h> #include <__type_traits/enable_if.h> #include <__type_traits/integral_constant.h> @@ -196,6 +197,57 @@ public: swap(__back_cap_, __back_capacity); } + /// Relocates the objects in the range `[__first, __last)` to the front of the buffer, then swaps + /// `__first`, `__last`, and `__capacity` with `__begin_`, `__end_`, and `__back_cap_`, + /// respectively. + /// + /// Precondition: `__front_spare() == __last - __first`. + /// Exceptions: This function has a strong exception guarantee. + _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void + __relocate(pointer& __first, pointer& __last, pointer& __capacity) { + auto __size = __last - __first; + auto __new_begin = __begin_ - __size; + std::__uninitialized_allocator_relocate( + __alloc_, std::__to_address(__first), std::__to_address(__last), std::__to_address(__new_begin)); + __begin_ = __new_begin; + __last = __first; + + __swap_layouts(__first, __last, __capacity); + __front_cap_ = __begin_; + } + + /// Relocates the objects in the range `[__first, __pivot)` to the front of the buffer, the + /// objects in `[__pivot, __last)` into the back of the buffer, then swaps `__first`, `__last`, + /// and `__capacity` with `__begin_`, `__end_`, and `__back_cap_`, respectively. + /// + /// Preconditions: + /// * `__front_spare() == __pivot - __first` + /// * `__back_spare() == __last - __pivot` + /// Exceptions: This function has a strong exception guarantee if `__first == __pivot` or + /// `__last == __pivot`. + _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI pointer + __relocate_with_pivot(pointer __pivot, pointer& __first, pointer& __last, pointer& __capacity) { + pointer __result = __begin_; + + // Relocate [__p, __last) first to avoid having a hole in [__first, __last) + // in case something in [__first, __p) throws. + std::__uninitialized_allocator_relocate( + __alloc_, std::__to_address(__pivot), std::__to_address(__last), std::__to_address(__end_)); + auto __relocated_so_far = __last - __pivot; + __end_ += __relocated_so_far; + __last = __pivot; // The objects in [__p, __last) have been destroyed by relocating them. + + auto __new_begin = __begin_ - (__pivot - __first); + std::__uninitialized_allocator_relocate( + __alloc_, std::__to_address(__first), std::__to_address(__pivot), std::__to_address(__new_begin)); + __begin_ = __new_begin; + + __last = __first; // All the objects have been destroyed by relocating them. + __swap_layouts(__first, __last, __capacity); + __front_cap_ = __begin_; + return __result; + } + private: pointer __front_cap_ = nullptr; pointer __begin_ = nullptr; @@ -352,6 +404,60 @@ public: swap(__cap_, __capacity); } + /// Relocates the objects in the range `[__first, __first + __n)` to the front of the buffer, then + /// swaps `__first`, `__n`, and `__capacity` with `__begin_`, `__size_`, and `__capacity_`, + /// respectively. + /// + /// Precondition: `__front_spare() == __n`. + /// Exceptions: This function has a strong exception guarantee. + _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void + __relocate(pointer& __first, size_type& __n, size_type& __capacity) { + auto __new_begin = __begin_ - __n; + auto __last = __first + __n; + std::__uninitialized_allocator_relocate( + __alloc_, std::__to_address(__first), std::__to_address(__last), std::__to_address(__new_begin)); + __set_valid_range(__new_begin, end()); + __n = 0; + + __swap_layouts(__first, __n, __capacity); + __front_cap_ = __begin_; + } + + /// Relocates the objects in the range `[__first, __pivot)` to the front of the buffer, the + /// objects in `[__pivot, __first + __n)` to the back of the buffer, then swaps `__first`, `__n`, + /// and `__capacity` with `__begin_`, `__size_`, and `__capacity_`, respectively. + /// + /// Preconditions: + /// * `__front_spare() == __pivot - __first` + /// * `__back_spare() == __last - __pivot` + /// Exceptions: This function has a strong exception guarantee if `__first == __pivot` or + /// `__first + __n == __pivot`. + _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI pointer + __relocate_with_pivot(pointer __pivot, pointer& __first, size_type& __n, size_type& __capacity) { + pointer __result = __begin_; + pointer __last = __first + __n; + + // We relocate elements in reverse order to ensure that a throwing relocation operation leaves the + // buffer in a valid state. + std::__uninitialized_allocator_relocate( + __alloc_, std::__to_address(__pivot), std::__to_address(__last), std::__to_address(end())); + + auto const __relocated_so_far = __last - __pivot; + __size_ += __relocated_so_far; + __n -= __relocated_so_far; // The objects in [__pivot, __last) have been destroyed by relocating them. + + auto __new_begin = __begin_ - __n; + std::__uninitialized_allocator_relocate( + __alloc_, std::__to_address(__first), std::__to_address(__pivot), std::__to_address(__new_begin)); + __begin_ = __new_begin; + __size_ += __n; + __n = 0; // All the objects have been destroyed by relocating them. + + __swap_layouts(__first, __n, __capacity); + __front_cap_ = __begin_; + return __result; + } + private: pointer __front_cap_ = nullptr; pointer __begin_ = nullptr; @@ -458,6 +564,8 @@ public: using __base_type::__get_allocator; using __base_type::__raw_capacity; using __base_type::__raw_sentinel; + using __base_type::__relocate; + using __base_type::__relocate_with_pivot; using __base_type::__reset; using __base_type::__set_capacity; using __base_type::__set_data; diff --git a/libcxx/include/__vector/layout.h b/libcxx/include/__vector/layout.h new file mode 100644 index 0000000000000..65d11aa238df0 --- /dev/null +++ b/libcxx/include/__vector/layout.h @@ -0,0 +1,514 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___VECTOR_LAYOUT_H +#define _LIBCPP___VECTOR_LAYOUT_H + +#include <__assert> +#include <__config> +#include <__debug_utils/sanitizers.h> +#include <__memory/allocator_traits.h> +#include <__memory/compressed_pair.h> +#include <__memory/pointer_traits.h> +#include <__memory/swap_allocator.h> +#include <__memory/uninitialized_algorithms.h> +#include <__split_buffer> +#include <__type_traits/is_nothrow_constructible.h> +#include <__utility/exchange.h> +#include <__utility/move.h> +#include <__utility/swap.h> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +_LIBCPP_PUSH_MACROS +#include <__undef_macros> + +_LIBCPP_BEGIN_NAMESPACE_STD + +/// Defines `std::vector`'s storage layout and any operations that are affected by a change in the +/// layout. +/// +/// `std::vector` can be represented in a variety of ways. Each representation strongly influences +/// the codegen when calling vector operations, which can significantly impact runtime performance +/// and memory utilisation. libc++ provides two alternative layouts for `std::vector`, although only +/// one can be active for an entire binary: +/// +/// * pointer-based layout (stable ABI default) +/// * size-based layout (unstable ABI alternative) +// +/// We describe these layouts below. All vector representations have a pointer that points to where +/// the memory is allocated (called `__begin_`). +/// +/// **Pointer-based layout** +/// +/// The pointer-based layout uses two more pointers in addition to `__begin_`. The second pointer +/// (called `__end_`) points past the end of the part of the buffer that holds valid elements. +/// Another pointer (called `__capacity_`) points past the end of the allocated buffer. The original +/// libc++ `std::vector` implementation only provided the pointer-based layout. libc++ continues to +/// use the pointer-based layout, by default, in order to maintain binary compatibility with +/// existing software. +/// +/// The `__end_` pointer has three primary use-cases: +/// * to compute the size of the vector; and +/// * to construct the past-the-end iterator; and +/// * to indicate where the next element should be appended. +/// +/// The `__capacity_` is used to compute the capacity of the vector, which lets the vector know how +/// many elements can be added to the vector before a reallocation is necessary. +/// +/// __begin_ = 0xE4FD0, __end_ = 0xE4FF0, __capacity_ = 0xE5000 +/// 0xE4FD0 0xE4FF0 0xE5000 +/// v v v +/// +---------------+--------+--------+--------+--------+--------+--------+---------------------+ +/// | ????????????? | 3174 | 5656 | 648 | 489 | ------ | ------ | ??????????????????? | +/// +---------------+--------+--------+--------+--------+--------+--------+---------------------+ +/// ^ ^ ^ +/// __begin_ __end_ __capacity_ +/// +/// Figure 1: A visual representation of a pointer-based `std::vector`. This vector has +/// four elements, with the capacity to store six. Boxes with numbers are valid elements within +/// the vector, and boxes with `xx` have been allocated, but aren't being used as elements right +/// now. +/// +/// This is the default layout for libc++. +/// +/// **Size-based layout** +/// +/// The size-based layout uses integers to track its size and capacity, and computes pointers to +/// past-the-end of the valid range and the whole buffer only when it's necessary. Programs using +/// the size-based layout have been measured to yield improved compute and memory performance over +/// the pointer-based layout. Despite these promising measurements, the size-based layout is opt-in, +/// to preserve ABI compatibility with prebuilt binaries. Given the improved performance, we +/// recommend preferring the size-based layout in the absence of such ABI constraints. +/// +/// __begin_ = 0xE4FD0, __size_ = 4, __capacity_ = 6 +/// 0xE4FD0 +/// v +/// +---------------+--------+--------+--------+--------+--------+--------+---------------------+ +/// | ????????????? | 3174 | 5656 | 648 | 489 | ------ | ------ | ??????????????????? | +/// +---------------+--------+--------+--------+--------+--------+--------+---------------------+ +/// ^ +/// __begin_ +/// +/// Figure 2: A visual representation of this a size-based layout. Blank boxes are not a part +/// of the vector's allocated buffer. +/// +/// **Class design** +/// +/// __vector_layout was designed with the following goals: +/// 1. to abstractly represent the buffer's boundaries; and +/// 2. to limit the number of `#ifdef` blocks that a reader needs to pass through; and +/// 3. given (1) and (2), to have no logically identical components in multiple `#ifdef` clauses. +/// +/// To facilitate these goals, there is a single `__vector_layout` definition. Users must choose +/// their vector's layout when libc++ is being configured, so there is no need to manage multiple +/// vector layout types (e.g. `__vector_size_layout`, `__vector_pointer_layout`, etc.). In doing so, +/// we reduce a significant portion of duplicate code. +template +class __vector_layout { +public: + using value_type _LIBCPP_NODEBUG = _Tp; + using allocator_type _LIBCPP_NODEBUG = _Allocator; + using __alloc_traits _LIBCPP_NODEBUG = allocator_traits; + using size_type _LIBCPP_NODEBUG = typename __alloc_traits::size_type; + using pointer _LIBCPP_NODEBUG = typename __alloc_traits::pointer; + using const_pointer _LIBCPP_NODEBUG = typename __alloc_traits::const_pointer; +#ifdef _LIBCPP_ABI_VECTOR_LAYOUT_SIZE_BASED + using _SplitBuffer _LIBCPP_NODEBUG = __split_buffer<_Tp, _Allocator, __split_buffer_size_layout>; + using __bound_type _LIBCPP_NODEBUG = size_type; +#else + using _SplitBuffer _LIBCPP_NODEBUG = __split_buffer<_Tp, _Allocator, __split_buffer_pointer_layout>; + using __bound_type _LIBCPP_NODEBUG = pointer; +#endif + + // Cannot be defaulted, since `_LIBCPP_COMPRESSED_PAIR` isn't an aggregate before C++14. + _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __vector_layout() + _NOEXCEPT_(is_nothrow_default_constructible::value) + : __capacity_() {} + + _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI explicit __vector_layout(allocator_type const& __a) + _NOEXCEPT_(is_nothrow_copy_constructible::value) + : __capacity_(), __alloc_(__a) {} + + _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI explicit __vector_layout(allocator_type&& __a) + _NOEXCEPT_(is_nothrow_move_constructible::value) + : __capacity_(), __alloc_(std::move(__a)) {} + + _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __vector_layout(__vector_layout&& __other) + _NOEXCEPT_(is_nothrow_move_constructible::value); + + /// Returns a reference to the stored allocator. + [[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI allocator_type& __alloc() _NOEXCEPT { + return __alloc_; + } + + [[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI allocator_type const& + __alloc() const _NOEXCEPT { + return __alloc_; + } + + /// Returns a pointer to the beginning of the buffer. + /// + /// `__begin_ptr()` is not called `data()` because `vector::data()` returns `T*`, but `__begin_` + /// is allowed to be a fancy pointer. + [[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI pointer __begin_ptr() _NOEXCEPT { + return __begin_; + } + + [[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_pointer __begin_ptr() const _NOEXCEPT { + return __begin_; + } + + /// Returns a built-in pointer to the beginning of the buffer. + [[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI _Tp* __data() _NOEXCEPT { + return std::__to_address(__begin_); + } + + [[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI _Tp const* __data() const _NOEXCEPT { + return std::__to_address(__begin_); + } + + /// Returns how many elements can be added before a reallocation occurs. + [[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type + __remaining_capacity() const _NOEXCEPT; + + /// Determines if a reallocation is necessary. + [[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI bool __is_full() const _NOEXCEPT; + + /// Sets the member pointing to the first element in the vector to `__new_begin`, the member used + /// to obtain the vector's bound to the equivalent of `__new_size`, and the member that represents + /// the vector's capacity to the equivalent to `__new_capacity`. + _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void + __set_layout(pointer __new_begin, size_type __new_size, size_type __new_capacity) _NOEXCEPT; + + /// Sets the member used to obtain the vector's bound to the equivalent of `__ptr`. + _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __set_bound_using_pointer(pointer __ptr) _NOEXCEPT; + + _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __reset_without_allocator() _NOEXCEPT; + _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __swap(__vector_layout& __other) _NOEXCEPT; + _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void + __move_assign_without_allocator(__vector_layout& __other) _NOEXCEPT; + + _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __relocate(_SplitBuffer& __buffer); + _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI pointer + __relocate_with_pivot(_SplitBuffer& __buffer, pointer __pivot); + + [[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type __size() const _NOEXCEPT; + [[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type __capacity() const _NOEXCEPT; + [[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI bool __empty() const _NOEXCEPT; + [[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI _Tp& __back() _NOEXCEPT; + [[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI _Tp const& __back() const _NOEXCEPT; + [[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI pointer __end_ptr() _NOEXCEPT; + [[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_pointer __end_ptr() const _NOEXCEPT; + [[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI pointer __capacity_ptr() _NOEXCEPT; + [[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_pointer __capacity_ptr() const _NOEXCEPT; + [[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI bool __invariants() const _NOEXCEPT; + +private: + pointer __begin_ = nullptr; + +#ifdef _LIBCPP_ABI_VECTOR_LAYOUT_SIZE_BASED + size_type __size_ = 0; + size_type __capacity_ = 0; + [[no_unique_address]] allocator_type __alloc_; +#else + pointer __end_ = nullptr; + _LIBCPP_COMPRESSED_PAIR(pointer, __capacity_ = nullptr, allocator_type, __alloc_); +#endif + + _LIBCPP_CONSTEXPR_SINCE_CXX20 + _LIBCPP_HIDE_FROM_ABI void __annotate_contiguous_container(const void* __old_mid, const void* __new_mid) const { + std::__annotate_contiguous_container<_Allocator>(__data(), __data() + __capacity(), __old_mid, __new_mid); + } + + _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __annotate_new(size_type __current_size) const _NOEXCEPT { + __annotate_contiguous_container(__data() + __capacity(), __data() + __current_size); + } + + _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __annotate_delete() const _NOEXCEPT { + __annotate_contiguous_container(__data() + __size(), __data() + __capacity()); + } +}; + +#ifdef _LIBCPP_ABI_VECTOR_LAYOUT_SIZE_BASED +template +_LIBCPP_CONSTEXPR_SINCE_CXX20 __vector_layout<_Tp, _Alloc>::__vector_layout(__vector_layout&& __other) + _NOEXCEPT_(is_nothrow_move_constructible::value) + : __begin_(std::__exchange(__other.__begin_, nullptr)), + __size_(std::__exchange(__other.__size_, 0)), + __capacity_(std::__exchange(__other.__capacity_, 0)), + __alloc_(std::move(__other.__alloc_)) {} + +template +_LIBCPP_CONSTEXPR_SINCE_CXX20 typename __vector_layout<_Tp, _Alloc>::size_type +__vector_layout<_Tp, _Alloc>::__remaining_capacity() const _NOEXCEPT { + return __capacity_ - __size_; +} + +template +_LIBCPP_CONSTEXPR_SINCE_CXX20 bool __vector_layout<_Tp, _Alloc>::__is_full() const _NOEXCEPT { + return __size_ == __capacity_; +} + +template +_LIBCPP_CONSTEXPR_SINCE_CXX20 void __vector_layout<_Tp, _Alloc>::__set_layout( + pointer __new_begin, size_type __new_size, size_type __new_capacity) _NOEXCEPT { + __begin_ = __new_begin; + __size_ = __new_size; + __capacity_ = __new_capacity; +} + +template +_LIBCPP_CONSTEXPR_SINCE_CXX20 void __vector_layout<_Tp, _Alloc>::__set_bound_using_pointer(pointer __ptr) _NOEXCEPT { + __size_ = static_cast(__ptr - __begin_); +} + +template +_LIBCPP_CONSTEXPR_SINCE_CXX20 void __vector_layout<_Tp, _Alloc>::__reset_without_allocator() _NOEXCEPT { + __begin_ = nullptr; + __size_ = 0; + __capacity_ = 0; +} + +template +_LIBCPP_CONSTEXPR_SINCE_CXX20 void __vector_layout<_Tp, _Alloc>::__swap(__vector_layout& __other) _NOEXCEPT { + using std::swap; + swap(__begin_, __other.__begin_); + swap(__size_, __other.__size_); + swap(__capacity_, __other.__capacity_); + std::__swap_allocator(__alloc_, __other.__alloc_); +} + +template +_LIBCPP_CONSTEXPR_SINCE_CXX20 void +__vector_layout<_Tp, _Alloc>::__move_assign_without_allocator(__vector_layout& __other) _NOEXCEPT { + __begin_ = __other.__begin_; + __size_ = __other.__size_; + __capacity_ = __other.__capacity_; + + __other.__reset_without_allocator(); +} + +template +_LIBCPP_CONSTEXPR_SINCE_CXX20 void __vector_layout<_Tp, _Alloc>::__relocate(_SplitBuffer& __buffer) { + __annotate_delete(); + __buffer.__relocate(__begin_, __size_, __capacity_); + __annotate_new(__size_); +} + +template +_LIBCPP_CONSTEXPR_SINCE_CXX20 typename __vector_layout<_Tp, _Alloc>::pointer +__vector_layout<_Tp, _Alloc>::__relocate_with_pivot(_SplitBuffer& __buffer, pointer __pivot) { + __annotate_delete(); + auto __result = __buffer.__relocate_with_pivot(__pivot, __begin_, __size_, __capacity_); + __annotate_new(__size_); + return __result; +} + +template +_LIBCPP_CONSTEXPR_SINCE_CXX20 typename __vector_layout<_Tp, _Alloc>::size_type +__vector_layout<_Tp, _Alloc>::__size() const _NOEXCEPT { + return __size_; +} + +template +_LIBCPP_CONSTEXPR_SINCE_CXX20 typename __vector_layout<_Tp, _Alloc>::size_type +__vector_layout<_Tp, _Alloc>::__capacity() const _NOEXCEPT { + return __capacity_; +} + +template +_LIBCPP_CONSTEXPR_SINCE_CXX20 bool __vector_layout<_Tp, _Alloc>::__empty() const _NOEXCEPT { + return __size_ == 0; +} + +template +_LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp& __vector_layout<_Tp, _Alloc>::__back() _NOEXCEPT { + return __begin_[__size_ - 1]; +} + +template +_LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp const& __vector_layout<_Tp, _Alloc>::__back() const _NOEXCEPT { + return __begin_[__size_ - 1]; +} + +template +_LIBCPP_CONSTEXPR_SINCE_CXX20 typename __vector_layout<_Tp, _Alloc>::pointer +__vector_layout<_Tp, _Alloc>::__end_ptr() _NOEXCEPT { + return __begin_ + __size_; +} + +template +_LIBCPP_CONSTEXPR_SINCE_CXX20 typename __vector_layout<_Tp, _Alloc>::const_pointer +__vector_layout<_Tp, _Alloc>::__end_ptr() const _NOEXCEPT { + return __begin_ + __size_; +} + +template +_LIBCPP_CONSTEXPR_SINCE_CXX20 typename __vector_layout<_Tp, _Alloc>::pointer +__vector_layout<_Tp, _Alloc>::__capacity_ptr() _NOEXCEPT { + return __begin_ + __capacity_; +} + +template +_LIBCPP_CONSTEXPR_SINCE_CXX20 typename __vector_layout<_Tp, _Alloc>::const_pointer +__vector_layout<_Tp, _Alloc>::__capacity_ptr() const _NOEXCEPT { + return __begin_ + __capacity_; +} + +template +_LIBCPP_CONSTEXPR_SINCE_CXX20 bool __vector_layout<_Tp, _Alloc>::__invariants() const _NOEXCEPT { + if (__begin_ == nullptr) + return __size_ == 0 && __capacity_ == 0; + return __size_ <= __capacity_; +} +#else // !defined(_LIBCPP_ABI_VECTOR_LAYOUT_SIZE_BASED) +template +_LIBCPP_CONSTEXPR_SINCE_CXX20 __vector_layout<_Tp, _Alloc>::__vector_layout(__vector_layout&& __other) + _NOEXCEPT_(is_nothrow_move_constructible::value) + : __begin_(std::__exchange(__other.__begin_, nullptr)), + __end_(std::__exchange(__other.__end_, nullptr)), + __capacity_(std::__exchange(__other.__capacity_, nullptr)), + __alloc_(std::move(__other.__alloc_)) {} + +template +_LIBCPP_CONSTEXPR_SINCE_CXX20 typename __vector_layout<_Tp, _Alloc>::size_type +__vector_layout<_Tp, _Alloc>::__remaining_capacity() const _NOEXCEPT { + return __capacity_ - __end_; +} + +template +[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 bool __vector_layout<_Tp, _Alloc>::__is_full() const _NOEXCEPT { + return __end_ == __capacity_; +} + +template +_LIBCPP_CONSTEXPR_SINCE_CXX20 void __vector_layout<_Tp, _Alloc>::__set_layout( + pointer __new_begin, size_type __new_size, size_type __new_capacity) _NOEXCEPT { + __begin_ = __new_begin; + __end_ = __new_begin + __new_size; + __capacity_ = __new_begin + __new_capacity; +} + +template +_LIBCPP_CONSTEXPR_SINCE_CXX20 void __vector_layout<_Tp, _Alloc>::__set_bound_using_pointer(pointer __ptr) _NOEXCEPT { + __end_ = __ptr; +} + +template +_LIBCPP_CONSTEXPR_SINCE_CXX20 void __vector_layout<_Tp, _Alloc>::__reset_without_allocator() _NOEXCEPT { + __begin_ = nullptr; + __end_ = nullptr; + __capacity_ = nullptr; +} + +template +_LIBCPP_CONSTEXPR_SINCE_CXX20 void __vector_layout<_Tp, _Alloc>::__swap(__vector_layout& __other) _NOEXCEPT { + using std::swap; + swap(__begin_, __other.__begin_); + swap(__end_, __other.__end_); + swap(__capacity_, __other.__capacity_); + std::__swap_allocator(__alloc_, __other.__alloc_); +} + +template +_LIBCPP_CONSTEXPR_SINCE_CXX20 void +__vector_layout<_Tp, _Alloc>::__move_assign_without_allocator(__vector_layout& __other) _NOEXCEPT { + __begin_ = __other.__begin_; + __end_ = __other.__end_; + __capacity_ = __other.__capacity_; + + __other.__reset_without_allocator(); +} + +template +_LIBCPP_CONSTEXPR_SINCE_CXX20 void __vector_layout<_Tp, _Alloc>::__relocate(_SplitBuffer& __buffer) { + __annotate_delete(); + __buffer.__relocate(__begin_, __end_, __capacity_); + __annotate_new(__size()); +} + +template +_LIBCPP_CONSTEXPR_SINCE_CXX20 typename __vector_layout<_Tp, _Alloc>::pointer +__vector_layout<_Tp, _Alloc>::__relocate_with_pivot(_SplitBuffer& __buffer, pointer __pivot) { + __annotate_delete(); + auto __result = __buffer.__relocate_with_pivot(__pivot, __begin_, __end_, __capacity_); + __annotate_new(__size()); + return __result; +} + +template +_LIBCPP_CONSTEXPR_SINCE_CXX20 typename __vector_layout<_Tp, _Alloc>::size_type +__vector_layout<_Tp, _Alloc>::__size() const _NOEXCEPT { + return static_cast(__end_ - __begin_); +} + +template +_LIBCPP_CONSTEXPR_SINCE_CXX20 typename __vector_layout<_Tp, _Alloc>::size_type +__vector_layout<_Tp, _Alloc>::__capacity() const _NOEXCEPT { + return static_cast(__capacity_ - __begin_); +} + +template +_LIBCPP_CONSTEXPR_SINCE_CXX20 bool __vector_layout<_Tp, _Alloc>::__empty() const _NOEXCEPT { + return __begin_ == __end_; +} + +template +_LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp& __vector_layout<_Tp, _Alloc>::__back() _NOEXCEPT { + return __end_[-1]; +} + +template +_LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp const& __vector_layout<_Tp, _Alloc>::__back() const _NOEXCEPT { + return __end_[-1]; +} + +template +_LIBCPP_CONSTEXPR_SINCE_CXX20 typename __vector_layout<_Tp, _Alloc>::pointer +__vector_layout<_Tp, _Alloc>::__end_ptr() _NOEXCEPT { + return __end_; +} + +template +_LIBCPP_CONSTEXPR_SINCE_CXX20 typename __vector_layout<_Tp, _Alloc>::const_pointer +__vector_layout<_Tp, _Alloc>::__end_ptr() const _NOEXCEPT { + return __end_; +} + +template +_LIBCPP_CONSTEXPR_SINCE_CXX20 typename __vector_layout<_Tp, _Alloc>::pointer +__vector_layout<_Tp, _Alloc>::__capacity_ptr() _NOEXCEPT { + return __capacity_; +} + +template +_LIBCPP_CONSTEXPR_SINCE_CXX20 typename __vector_layout<_Tp, _Alloc>::const_pointer +__vector_layout<_Tp, _Alloc>::__capacity_ptr() const _NOEXCEPT { + return __capacity_; +} + +template +_LIBCPP_CONSTEXPR_SINCE_CXX20 bool __vector_layout<_Tp, _Alloc>::__invariants() const _NOEXCEPT { + if (__begin_ == nullptr) + return __end_ == nullptr && __capacity_ == nullptr; + if (__begin_ > __end_) + return false; + if (__begin_ == __capacity_) + return false; + return __end_ <= __capacity_; +} +#endif // _LIBCPP_ABI_SIZE_BASED_VECTOR + +_LIBCPP_END_NAMESPACE_STD + +_LIBCPP_POP_MACROS + +#endif // _LIBCPP___VECTOR_LAYOUT_H diff --git a/libcxx/include/__vector/vector.h b/libcxx/include/__vector/vector.h index 2a081e3cdb1e2..2b9508ecafeac 100644 --- a/libcxx/include/__vector/vector.h +++ b/libcxx/include/__vector/vector.h @@ -56,6 +56,7 @@ #include <__type_traits/is_nothrow_constructible.h> #include <__type_traits/is_pointer.h> #include <__type_traits/is_same.h> +#include <__type_traits/is_swappable.h> #include <__type_traits/is_trivially_relocatable.h> #include <__type_traits/type_identity.h> #include <__utility/declval.h> @@ -72,6 +73,7 @@ // These headers define parts of vectors definition, since they define ADL functions or class specializations. #include <__vector/comparison.h> #include <__vector/container_traits.h> +#include <__vector/layout.h> #include <__vector/swap.h> #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) @@ -85,7 +87,9 @@ _LIBCPP_BEGIN_NAMESPACE_STD template */> class vector { - using _SplitBuffer _LIBCPP_NODEBUG = std::__split_buffer<_Tp, _Allocator, __split_buffer_pointer_layout>; + using __base_type _LIBCPP_NODEBUG = __vector_layout<_Tp, _Allocator>; + using __bound_type _LIBCPP_NODEBUG = typename __base_type::__bound_type; + using _SplitBuffer _LIBCPP_NODEBUG = typename __base_type::_SplitBuffer; public: // @@ -93,9 +97,9 @@ class vector { // using value_type = _Tp; using allocator_type = _Allocator; - using __alloc_traits _LIBCPP_NODEBUG = allocator_traits; - using reference = value_type&; - using const_reference = const value_type&; + using __alloc_traits _LIBCPP_NODEBUG = allocator_traits<_Allocator>; + using reference = _Tp&; + using const_reference = const _Tp&; using size_type = typename __alloc_traits::size_type; using difference_type = typename __alloc_traits::difference_type; using pointer = typename __alloc_traits::pointer; @@ -137,7 +141,7 @@ class vector { #else noexcept #endif - : __alloc_(__a) { + : __layout_(__a) { } _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI explicit vector(size_type __n) { @@ -151,7 +155,7 @@ class vector { #if _LIBCPP_STD_VER >= 14 _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI explicit vector(size_type __n, const allocator_type& __a) - : __alloc_(__a) { + : __layout_(__a) { auto __guard = std::__make_exception_guard(__destroy_vector(*this)); if (__n > 0) { __vallocate(__n); @@ -173,7 +177,7 @@ class vector { template <__enable_if_t<__is_allocator_v<_Allocator>, int> = 0> _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI vector(size_type __n, const value_type& __x, const allocator_type& __a) - : __alloc_(__a) { + : __layout_(__a) { auto __guard = std::__make_exception_guard(__destroy_vector(*this)); if (__n > 0) { __vallocate(__n); @@ -196,7 +200,7 @@ class vector { int> = 0> _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI vector(_InputIterator __first, _InputIterator __last, const allocator_type& __a) - : __alloc_(__a) { + : __layout_(__a) { __init_with_sentinel(__first, __last); } @@ -217,16 +221,15 @@ class vector { int> = 0> _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI vector(_ForwardIterator __first, _ForwardIterator __last, const allocator_type& __a) - : __alloc_(__a) { + : __layout_(__a) { size_type __n = static_cast(std::distance(__first, __last)); __init_with_size(__first, __last, __n); } #if _LIBCPP_STD_VER >= 23 template <_ContainerCompatibleRange<_Tp> _Range> - _LIBCPP_HIDE_FROM_ABI constexpr vector( - from_range_t, _Range&& __range, const allocator_type& __alloc = allocator_type()) - : __alloc_(__alloc) { + _LIBCPP_HIDE_FROM_ABI constexpr vector(from_range_t, _Range&& __range, const allocator_type& __a = allocator_type()) + : __layout_(__a) { if constexpr (ranges::forward_range<_Range> || ranges::sized_range<_Range>) { auto __n = static_cast(ranges::distance(__range)); __init_with_size(ranges::begin(__range), ranges::end(__range), __n); @@ -243,10 +246,10 @@ class vector { _LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI __destroy_vector(vector& __vec) : __vec_(__vec) {} _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void operator()() { - if (__vec_.__begin_ != nullptr) { + if (__vec_.__layout_.__begin_ptr() != nullptr) { __vec_.clear(); __vec_.__annotate_delete(); - __alloc_traits::deallocate(__vec_.__alloc_, __vec_.__begin_, __vec_.capacity()); + __alloc_traits::deallocate(__vec_.__layout_.__alloc(), __vec_.__layout_.__begin_ptr(), __vec_.capacity()); } } @@ -254,17 +257,19 @@ class vector { vector& __vec_; }; + using __emplace_back_result_t _LIBCPP_NODEBUG = _If<(_LIBCPP_STD_VER < 17), void, reference>; + public: _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI ~vector() { __destroy_vector (*this)(); } _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI vector(const vector& __x) - : __alloc_(__alloc_traits::select_on_container_copy_construction(__x.__alloc_)) { - __init_with_size(__x.__begin_, __x.__end_, __x.size()); + : __layout_(__alloc_traits::select_on_container_copy_construction(__x.__layout_.__alloc())) { + __init_with_size(__x.__layout_.__begin_ptr(), __x.__layout_.__end_ptr(), __x.size()); } _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI vector(const vector& __x, const __type_identity_t& __a) - : __alloc_(__a) { - __init_with_size(__x.__begin_, __x.__end_, __x.size()); + : __layout_(__a) { + __init_with_size(__x.__layout_.__begin_ptr(), __x.__layout_.__end_ptr(), __x.size()); } _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI vector& operator=(const vector& __x); @@ -275,7 +280,7 @@ class vector { _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI vector(initializer_list __il, const allocator_type& __a) - : __alloc_(__a) { + : __layout_(__a) { __init_with_size(__il.begin(), __il.end(), __il.size()); } @@ -338,23 +343,23 @@ class vector { #endif [[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI allocator_type get_allocator() const _NOEXCEPT { - return this->__alloc_; + return this->__layout_.__alloc(); } // // Iterators // [[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator begin() _NOEXCEPT { - return __make_iter(__add_alignment_assumption(this->__begin_)); + return __make_iter(__add_alignment_assumption(this->__layout_.__begin_ptr())); } [[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_iterator begin() const _NOEXCEPT { - return __make_iter(__add_alignment_assumption(this->__begin_)); + return __make_iter(__add_alignment_assumption(this->__layout_.__begin_ptr())); } [[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator end() _NOEXCEPT { - return __make_iter(__add_alignment_assumption(this->__end_)); + return __make_iter(__add_alignment_assumption(__layout_.__end_ptr())); } [[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_iterator end() const _NOEXCEPT { - return __make_iter(__add_alignment_assumption(this->__end_)); + return __make_iter(__add_alignment_assumption(__layout_.__end_ptr())); } [[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI reverse_iterator rbegin() _NOEXCEPT { @@ -389,16 +394,17 @@ class vector { // [vector.capacity], capacity // [[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type size() const _NOEXCEPT { - return static_cast(this->__end_ - this->__begin_); + return __layout_.__size(); } [[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type capacity() const _NOEXCEPT { - return static_cast(this->__cap_ - this->__begin_); + return __layout_.__capacity(); } [[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI bool empty() const _NOEXCEPT { - return this->__begin_ == this->__end_; + return __layout_.__empty(); } + [[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT { - return std::min(__alloc_traits::max_size(this->__alloc_), numeric_limits::max()); + return std::min(__alloc_traits::max_size(__layout_.__alloc()), numeric_limits::max()); } _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void reserve(size_type __n); _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void shrink_to_fit() _NOEXCEPT; @@ -408,50 +414,50 @@ class vector { // [[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI reference operator[](size_type __n) _NOEXCEPT { _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__n < size(), "vector[] index out of bounds"); - return this->__begin_[__n]; + return this->__layout_.__begin_ptr()[__n]; } [[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reference operator[](size_type __n) const _NOEXCEPT { _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__n < size(), "vector[] index out of bounds"); - return this->__begin_[__n]; + return this->__layout_.__begin_ptr()[__n]; } [[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI reference at(size_type __n) { if (__n >= size()) this->__throw_out_of_range(); - return this->__begin_[__n]; + return this->__layout_.__begin_ptr()[__n]; } [[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reference at(size_type __n) const { if (__n >= size()) this->__throw_out_of_range(); - return this->__begin_[__n]; + return this->__layout_.__begin_ptr()[__n]; } [[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI reference front() _NOEXCEPT { _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "front() called on an empty vector"); - return *this->__begin_; + return *this->__layout_.__begin_ptr(); } [[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reference front() const _NOEXCEPT { _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "front() called on an empty vector"); - return *this->__begin_; + return *this->__layout_.__begin_ptr(); } [[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI reference back() _NOEXCEPT { _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "back() called on an empty vector"); - return *(this->__end_ - 1); + return __layout_.__back(); } [[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reference back() const _NOEXCEPT { _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "back() called on an empty vector"); - return *(this->__end_ - 1); + return __layout_.__back(); } // // [vector.data], data access // [[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI value_type* data() _NOEXCEPT { - return std::__to_address(this->__begin_); + return __layout_.__data(); } [[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const value_type* data() const _NOEXCEPT { - return std::__to_address(this->__begin_); + return __layout_.__data(); } // @@ -462,19 +468,15 @@ class vector { _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void push_back(value_type&& __x) { emplace_back(std::move(__x)); } template - _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI -#if _LIBCPP_STD_VER >= 17 - reference emplace_back(_Args&&... __args); -#else - void emplace_back(_Args&&... __args); -#endif + _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __emplace_back_result_t emplace_back(_Args&&... __args); template _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __emplace_back_assume_capacity(_Args&&... __args) { _LIBCPP_ASSERT_INTERNAL( size() < capacity(), "We assume that we have enough space to insert an element at the end of the vector"); _ConstructTransaction __tx(*this, 1); - __alloc_traits::construct(this->__alloc_, std::__to_address(__tx.__pos_), std::forward<_Args>(__args)...); + __alloc_traits::construct( + this->__layout_.__alloc(), std::__to_address(__tx.__pos_), std::forward<_Args>(__args)...); ++__tx.__pos_; } @@ -483,15 +485,15 @@ class vector { _LIBCPP_HIDE_FROM_ABI constexpr void append_range(_Range&& __range) { if constexpr (ranges::forward_range<_Range> || ranges::sized_range<_Range>) { auto __len = ranges::distance(__range); - if (__len <= __cap_ - __end_) { + if (__len <= static_cast(__layout_.__remaining_capacity())) { __construct_at_end(ranges::begin(__range), ranges::end(__range), __len); } else { - _SplitBuffer __buffer(__recommend(size() + __len), size(), __alloc_); + _SplitBuffer __buffer(__recommend(size() + __len), size(), __layout_.__alloc()); __buffer.__construct_at_end_with_size(ranges::begin(__range), __len); - __swap_out_circular_buffer(__buffer); + __layout_.__relocate(__buffer); } } else { - vector __buffer(__alloc_); + vector __buffer(__layout_.__alloc()); for (auto&& __val : __range) __buffer.emplace_back(std::forward(__val)); append_range(ranges::as_rvalue_view(__buffer)); @@ -501,7 +503,7 @@ class vector { _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void pop_back() { _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "vector::pop_back called on an empty vector"); - this->__destruct_at_end(this->__end_ - 1); + this->__destruct_at_end(__layout_.__end_ptr() - 1); } _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __position, const_reference __x); @@ -557,7 +559,7 @@ class vector { _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void clear() _NOEXCEPT { size_type __old_size = size(); - __base_destruct_at_end(this->__begin_); + __base_destruct_at_end(this->__layout_.__begin_ptr()); __annotate_shrink(__old_size); } @@ -571,27 +573,41 @@ class vector { _NOEXCEPT_(!__alloc_traits::propagate_on_container_swap::value || __is_nothrow_swappable_v); #endif - _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI bool __invariants() const; + _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI bool __invariants() const _NOEXCEPT { + return __layout_.__invariants(); + } private: - pointer __begin_ = nullptr; - pointer __end_ = nullptr; - _LIBCPP_COMPRESSED_PAIR(pointer, __cap_ = nullptr, allocator_type, __alloc_); + __base_type __layout_; // Allocate space for __n objects // throws length_error if __n > max_size() // throws (probably bad_alloc) if memory run out - // Precondition: __begin_ == __end_ == __cap_ == nullptr + // Precondition: begin() == nullptr + // Precondition: size() == 0 + // Precondition: capacity() == 0 // Precondition: __n > 0 // Postcondition: capacity() >= __n // Postcondition: size() == 0 _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __vallocate(size_type __n) { + _LIBCPP_ASSERT_INTERNAL( + __layout_.__begin_ptr() == nullptr, + "vector::__vallocate can only be called on a vector that hasn't allocated memory. This vector either already " + "owns a buffer, or a deallocation function didn't reset the layout's begin pointer."); + _LIBCPP_ASSERT_INTERNAL( + __layout_.__empty(), + "vector::__vallocate can only be called on a vector that hasn't allocated memory. This vector either already " + "owns a buffer, or a deallocation function didn't reset the layout's size."); + _LIBCPP_ASSERT_INTERNAL( + __layout_.__capacity() == 0, + "vector::__vallocate can only be called on a vector that hasn't allocated memory. This vector either already " + "owns a buffer, or a deallocation function didn't reset the layout's capacity."); + _LIBCPP_ASSERT_INTERNAL(__n > 0, "vector::__vallocate cannot allocate 0 bytes"); + if (__n > max_size()) this->__throw_length_error(); - auto __allocation = std::__allocate_at_least(this->__alloc_, __n); - __begin_ = __allocation.ptr; - __end_ = __allocation.ptr; - __cap_ = __begin_ + __allocation.count; + auto __allocation = std::__allocate_at_least(this->__layout_.__alloc(), __n); + __layout_.__set_layout(__allocation.ptr, 0, __allocation.count); __annotate_new(0); } @@ -640,7 +656,7 @@ class vector { _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __insert_assign_n_unchecked(_Iterator __first, difference_type __n, pointer __position) { for (pointer __end_position = __position + __n; __position != __end_position; ++__position, (void)++__first) { - __temp_value __tmp(this->__alloc_, *__first); + __temp_value __tmp(this->__layout_.__alloc(), *__first); *__position = std::move(__tmp.get()); } } @@ -676,7 +692,7 @@ class vector { // current implementation, there is no connection between a bounded iterator and its associated container, so we // don't have a way to update existing valid iterators when the container is resized and thus have to go with // a laxer approach. - return std::__make_bounded_iter(__p, this->__begin_, this->__cap_); + return std::__make_bounded_iter(__p, __layout_.__begin_ptr(), __layout_.__capacity_ptr()); #else return iterator(__p); #endif // _LIBCPP_ABI_BOUNDED_ITERATORS_IN_VECTOR @@ -685,15 +701,12 @@ class vector { _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_iterator __make_iter(const_pointer __p) const _NOEXCEPT { #ifdef _LIBCPP_ABI_BOUNDED_ITERATORS_IN_VECTOR // Bound the iterator according to the capacity, rather than the size. - return std::__make_bounded_iter(__p, const_pointer(this->__begin_), const_pointer(this->__cap_)); + return std::__make_bounded_iter(__p, __layout_.__begin_ptr(), __layout_.__capacity_ptr()); #else return const_iterator(__p); #endif // _LIBCPP_ABI_BOUNDED_ITERATORS_IN_VECTOR } - _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __swap_out_circular_buffer(_SplitBuffer& __v); - _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI pointer - __swap_out_circular_buffer(_SplitBuffer& __v, pointer __p); _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __move_range(pointer __from_s, pointer __from_e, pointer __to); _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __move_assign(vector& __c, true_type) @@ -740,14 +753,14 @@ class vector { struct _ConstructTransaction { _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI explicit _ConstructTransaction(vector& __v, size_type __n) - : __v_(__v), __pos_(__v.__end_), __new_end_(__v.__end_ + __n) { + : __v_(__v), __pos_(__v.__layout_.__end_ptr()), __new_end_(__pos_ + __n) { __v_.__annotate_increase(__n); } _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI ~_ConstructTransaction() { - __v_.__end_ = __pos_; + __v_.__layout_.__set_bound_using_pointer(__pos_); if (__pos_ != __new_end_) { - __v_.__annotate_shrink(__new_end_ - __v_.__begin_); + __v_.__annotate_shrink(__new_end_ - __v_.__layout_.__begin_ptr()); } } @@ -760,10 +773,10 @@ class vector { }; _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __base_destruct_at_end(pointer __new_last) _NOEXCEPT { - pointer __soon_to_be_end = this->__end_; + pointer __soon_to_be_end = __layout_.__end_ptr(); while (__new_last != __soon_to_be_end) - __alloc_traits::destroy(this->__alloc_, std::__to_address(--__soon_to_be_end)); - this->__end_ = __new_last; + __alloc_traits::destroy(this->__layout_.__alloc(), std::__to_address(--__soon_to_be_end)); + __layout_.__set_bound_using_pointer(__new_last); } _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __copy_assign_alloc(const vector& __c) { @@ -781,20 +794,20 @@ class vector { [[__noreturn__]] _LIBCPP_HIDE_FROM_ABI static void __throw_out_of_range() { std::__throw_out_of_range("vector"); } _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __copy_assign_alloc(const vector& __c, true_type) { - if (this->__alloc_ != __c.__alloc_) { + if (this->__layout_.__alloc() != __c.__layout_.__alloc()) { clear(); __annotate_delete(); - __alloc_traits::deallocate(this->__alloc_, this->__begin_, capacity()); - this->__begin_ = this->__end_ = this->__cap_ = nullptr; + __alloc_traits::deallocate(this->__layout_.__alloc(), this->__layout_.__begin_ptr(), capacity()); + __layout_.__reset_without_allocator(); } - this->__alloc_ = __c.__alloc_; + this->__layout_.__alloc() = __c.__layout_.__alloc(); } _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __copy_assign_alloc(const vector&, false_type) {} _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __move_assign_alloc(vector& __c, true_type) _NOEXCEPT_(is_nothrow_move_assignable::value) { - this->__alloc_ = std::move(__c.__alloc_); + this->__layout_.__alloc() = std::move(__c.__layout_.__alloc()); } _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __move_assign_alloc(vector&, false_type) _NOEXCEPT {} @@ -803,7 +816,7 @@ class vector { static _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI _LIBCPP_NO_CFI _Ptr __add_alignment_assumption(_Ptr __p) _NOEXCEPT { if (!__libcpp_is_constant_evaluated()) { - return static_cast(__builtin_assume_aligned(__p, _LIBCPP_ALIGNOF(decltype(*__p)))); + return static_cast<_Ptr>(__builtin_assume_aligned(__p, _LIBCPP_ALIGNOF(decltype(*__p)))); } return __p; } @@ -813,10 +826,6 @@ class vector { __add_alignment_assumption(_Ptr __p) _NOEXCEPT { return __p; } - - _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __swap_layouts(_SplitBuffer& __sb) { - __sb.__swap_layouts(__begin_, __end_, __cap_); - } }; #if _LIBCPP_STD_VER >= 17 @@ -840,59 +849,13 @@ template vector, _Alloc>; #endif -// __swap_out_circular_buffer relocates the objects in [__begin_, __end_) into the front of __v and swaps the buffers of -// *this and __v. It is assumed that __v provides space for exactly (__end_ - __begin_) objects in the front. This -// function has a strong exception guarantee. -template -_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::__swap_out_circular_buffer(_SplitBuffer& __v) { - __annotate_delete(); - auto __new_begin = __v.begin() - size(); - std::__uninitialized_allocator_relocate( - this->__alloc_, std::__to_address(__begin_), std::__to_address(__end_), std::__to_address(__new_begin)); - __v.__set_valid_range(__new_begin, __v.end()); - __end_ = __begin_; // All the objects have been destroyed by relocating them. - - __swap_layouts(__v); - __v.__set_data(__v.begin()); - __annotate_new(size()); -} - -// __swap_out_circular_buffer relocates the objects in [__begin_, __p) into the front of __v, the objects in -// [__p, __end_) into the back of __v and swaps the buffers of *this and __v. It is assumed that __v provides space for -// exactly (__p - __begin_) objects in the front and space for at least (__end_ - __p) objects in the back. This -// function has a strong exception guarantee if __begin_ == __p || __end_ == __p. -template -_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::pointer -vector<_Tp, _Allocator>::__swap_out_circular_buffer(_SplitBuffer& __v, pointer __p) { - __annotate_delete(); - pointer __ret = __v.begin(); - - // Relocate [__p, __end_) first to avoid having a hole in [__begin_, __end_) - // in case something in [__begin_, __p) throws. - std::__uninitialized_allocator_relocate( - this->__alloc_, std::__to_address(__p), std::__to_address(__end_), std::__to_address(__v.end())); - auto __relocated_so_far = __end_ - __p; - __v.__set_sentinel(__v.end() + __relocated_so_far); - __end_ = __p; // The objects in [__p, __end_) have been destroyed by relocating them. - auto __new_begin = __v.begin() - (__p - __begin_); - - std::__uninitialized_allocator_relocate( - this->__alloc_, std::__to_address(__begin_), std::__to_address(__p), std::__to_address(__new_begin)); - __v.__set_valid_range(__new_begin, __v.end()); - __end_ = __begin_; // All the objects have been destroyed by relocating them. - __swap_layouts(__v); - __v.__set_data(__v.begin()); - __annotate_new(size()); - return __ret; -} - template _LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::__vdeallocate() _NOEXCEPT { - if (this->__begin_ != nullptr) { + if (this->__layout_.__begin_ptr() != nullptr) { clear(); __annotate_delete(); - __alloc_traits::deallocate(this->__alloc_, this->__begin_, capacity()); - this->__begin_ = this->__end_ = this->__cap_ = nullptr; + __alloc_traits::deallocate(this->__layout_.__alloc(), this->__layout_.__begin_ptr(), capacity()); + __layout_.__reset_without_allocator(); } } @@ -909,7 +872,7 @@ vector<_Tp, _Allocator>::__recommend(size_type __new_size) const { return std::max(2 * __cap, __new_size); } -// Default constructs __n objects starting at __end_ +// Default constructs __n objects starting at __layout_.__end_ptr() // throws if construction throws // Precondition: __n > 0 // Precondition: size() + __n <= capacity() @@ -919,11 +882,11 @@ _LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::__construct_at_end(s _ConstructTransaction __tx(*this, __n); const_pointer __new_end = __tx.__new_end_; for (pointer __pos = __tx.__pos_; __pos != __new_end; __tx.__pos_ = ++__pos) { - __alloc_traits::construct(this->__alloc_, std::__to_address(__pos)); + __alloc_traits::construct(this->__layout_.__alloc(), std::__to_address(__pos)); } } -// Copy constructs __n objects starting at __end_ from __x +// Copy constructs __n objects starting at __layout_.__end_ptr() from __x // throws if construction throws // Precondition: __n > 0 // Precondition: size() + __n <= capacity() @@ -935,7 +898,7 @@ vector<_Tp, _Allocator>::__construct_at_end(size_type __n, const_reference __x) _ConstructTransaction __tx(*this, __n); const_pointer __new_end = __tx.__new_end_; for (pointer __pos = __tx.__pos_; __pos != __new_end; __tx.__pos_ = ++__pos) { - __alloc_traits::construct(this->__alloc_, std::__to_address(__pos), __x); + __alloc_traits::construct(this->__layout_.__alloc(), std::__to_address(__pos), __x); } } @@ -944,7 +907,8 @@ template _LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::__construct_at_end(_InputIterator __first, _Sentinel __last, size_type __n) { _ConstructTransaction __tx(*this, __n); - __tx.__pos_ = std::__uninitialized_allocator_copy(this->__alloc_, std::move(__first), std::move(__last), __tx.__pos_); + __tx.__pos_ = std::__uninitialized_allocator_copy( + this->__layout_.__alloc(), std::move(__first), std::move(__last), __tx.__pos_); } template @@ -954,22 +918,15 @@ _LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI vector<_Tp, _Allocato #else _NOEXCEPT_(is_nothrow_move_constructible::value) #endif - : __alloc_(std::move(__x.__alloc_)) { - this->__begin_ = __x.__begin_; - this->__end_ = __x.__end_; - this->__cap_ = __x.__cap_; - __x.__begin_ = __x.__end_ = __x.__cap_ = nullptr; + : __layout_(std::move(__x.__layout_)) { } template _LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI vector<_Tp, _Allocator>::vector(vector&& __x, const __type_identity_t& __a) - : __alloc_(__a) { - if (__a == __x.__alloc_) { - this->__begin_ = __x.__begin_; - this->__end_ = __x.__end_; - this->__cap_ = __x.__cap_; - __x.__begin_ = __x.__end_ = __x.__cap_ = nullptr; + : __layout_(__a) { + if (__a == __x.__layout_.__alloc()) { + __layout_.__move_assign_without_allocator(__x.__layout_); } else { typedef move_iterator _Ip; __init_with_size(_Ip(__x.begin()), _Ip(__x.end()), __x.size()); @@ -979,7 +936,7 @@ vector<_Tp, _Allocator>::vector(vector&& __x, const __type_identity_t _LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::__move_assign(vector& __c, false_type) _NOEXCEPT_(__alloc_traits::is_always_equal::value) { - if (this->__alloc_ != __c.__alloc_) { + if (this->__layout_.__alloc() != __c.__layout_.__alloc()) { typedef move_iterator _Ip; assign(_Ip(__c.begin()), _Ip(__c.end())); } else @@ -991,10 +948,7 @@ _LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::__move_assign(vector _NOEXCEPT_(is_nothrow_move_assignable::value) { __vdeallocate(); __move_assign_alloc(__c); // this can throw - this->__begin_ = __c.__begin_; - this->__end_ = __c.__end_; - this->__cap_ = __c.__cap_; - __c.__begin_ = __c.__end_ = __c.__cap_ = nullptr; + __layout_.__move_assign_without_allocator(__c.__layout_); } template @@ -1002,7 +956,7 @@ _LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI vector<_Tp, _Allocato vector<_Tp, _Allocator>::operator=(const vector& __x) { if (this != std::addressof(__x)) { __copy_assign_alloc(__x); - assign(__x.__begin_, __x.__end_); + assign(__x.__layout_.__begin_ptr(), __x.__layout_.__end_ptr()); } return *this; } @@ -1011,10 +965,11 @@ template template _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void vector<_Tp, _Allocator>::__assign_with_sentinel(_Iterator __first, _Sentinel __last) { - pointer __cur = __begin_; - for (; __first != __last && __cur != __end_; ++__first, (void)++__cur) + pointer __cur = __layout_.__begin_ptr(); + pointer __end = __layout_.__end_ptr(); + for (; __first != __last && __cur != __end; ++__first, (void)++__cur) *__cur = *__first; - if (__cur != __end_) { + if (__cur != __end) { __destruct_at_end(__cur); } else { for (; __first != __last; ++__first) @@ -1030,10 +985,10 @@ vector<_Tp, _Allocator>::__assign_with_size(_Iterator __first, _Sentinel __last, if (__new_size <= capacity()) { auto const __size = size(); if (__new_size > __size) { - auto __mid = std::__copy_n<_AlgPolicy>(std::move(__first), __size, this->__begin_).__in_; + auto __mid = std::__copy_n<_AlgPolicy>(std::move(__first), __size, this->__layout_.__begin_ptr()).__in_; __construct_at_end(std::move(__mid), std::move(__last), __new_size - __size); } else { - pointer __m = std::__copy(std::move(__first), __last, this->__begin_).__out_; + pointer __m = std::__copy(std::move(__first), __last, this->__layout_.__begin_ptr()).__out_; this->__destruct_at_end(__m); } } else { @@ -1047,11 +1002,11 @@ template _LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::assign(size_type __n, const_reference __u) { if (__n <= capacity()) { size_type __s = size(); - std::fill_n(this->__begin_, std::min(__n, __s), __u); + std::fill_n(this->__layout_.__begin_ptr(), std::min(__n, __s), __u); if (__n > __s) __construct_at_end(__n - __s, __u); else - this->__destruct_at_end(this->__begin_ + __n); + this->__destruct_at_end(this->__layout_.__begin_ptr() + __n); } else { __vdeallocate(); __vallocate(__recommend(static_cast(__n))); @@ -1064,8 +1019,8 @@ _LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::reserve(size_type __ if (__n > capacity()) { if (__n > max_size()) this->__throw_length_error(); - _SplitBuffer __v(__n, size(), this->__alloc_); - __swap_out_circular_buffer(__v); + _SplitBuffer __v(__n, size(), this->__layout_.__alloc()); + __layout_.__relocate(__v); } } @@ -1075,12 +1030,12 @@ _LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::shrink_to_fit() _NOE #if _LIBCPP_HAS_EXCEPTIONS try { #endif // _LIBCPP_HAS_EXCEPTIONS - _SplitBuffer __v(size(), size(), this->__alloc_); + _SplitBuffer __v(size(), size(), this->__layout_.__alloc()); // The Standard mandates shrink_to_fit() does not increase the capacity. // With equal capacity keep the existing buffer. This avoids extra work // due to swapping the elements. if (__v.capacity() < capacity()) - __swap_out_circular_buffer(__v); + __layout_.__relocate(__v); #if _LIBCPP_HAS_EXCEPTIONS } catch (...) { } @@ -1092,13 +1047,13 @@ template template _LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::pointer vector<_Tp, _Allocator>::__emplace_back_slow_path(_Args&&... __args) { - _SplitBuffer __v(__recommend(size() + 1), size(), this->__alloc_); + _SplitBuffer __v(__recommend(size() + 1), size(), this->__layout_.__alloc()); // __v.emplace_back(std::forward<_Args>(__args)...); pointer __end = __v.end(); - __alloc_traits::construct(this->__alloc_, std::__to_address(__end), std::forward<_Args>(__args)...); + __alloc_traits::construct(this->__layout_.__alloc(), std::__to_address(__end), std::forward<_Args>(__args)...); __v.__set_sentinel(++__end); - __swap_out_circular_buffer(__v); - return this->__end_; + __layout_.__relocate(__v); + return __end; } // This makes the compiler inline `__else()` if `__cond` is known to be false. Currently LLVM doesn't do that without @@ -1119,27 +1074,22 @@ _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void __if_likely_else(bool _ } } -template +template template -_LIBCPP_CONSTEXPR_SINCE_CXX20 inline -#if _LIBCPP_STD_VER >= 17 - typename vector<_Tp, _Allocator>::reference -#else - void -#endif - vector<_Tp, _Allocator>::emplace_back(_Args&&... __args) { - pointer __end = this->__end_; +_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI typename vector<_Tp, _Alloc>::__emplace_back_result_t +vector<_Tp, _Alloc>::emplace_back(_Args&&... __args) { + pointer __end = __layout_.__end_ptr(); std::__if_likely_else( - __end < this->__cap_, + !__layout_.__is_full(), [&] { __emplace_back_assume_capacity(std::forward<_Args>(__args)...); ++__end; }, [&] { __end = __emplace_back_slow_path(std::forward<_Args>(__args)...); }); - this->__end_ = __end; + __layout_.__set_bound_using_pointer(__end); #if _LIBCPP_STD_VER >= 17 - return *(__end - 1); + return back(); #endif } @@ -1149,8 +1099,8 @@ vector<_Tp, _Allocator>::erase(const_iterator __position) { _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS( __position != end(), "vector::erase(iterator) called with a non-dereferenceable iterator"); difference_type __ps = __position - cbegin(); - pointer __p = this->__begin_ + __ps; - this->__destruct_at_end(std::move(__p + 1, this->__end_, __p)); + pointer __p = this->__layout_.__begin_ptr() + __ps; + this->__destruct_at_end(std::move(__p + 1, __layout_.__end_ptr(), __p)); return __make_iter(__p); } @@ -1158,9 +1108,9 @@ template _LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::iterator vector<_Tp, _Allocator>::erase(const_iterator __first, const_iterator __last) { _LIBCPP_ASSERT_VALID_INPUT_RANGE(__first <= __last, "vector::erase(first, last) called with invalid range"); - pointer __p = this->__begin_ + (__first - begin()); + pointer __p = this->__layout_.__begin_ptr() + (__first - begin()); if (__first != __last) { - this->__destruct_at_end(std::move(__p + (__last - __first), this->__end_, __p)); + this->__destruct_at_end(std::move(__p + (__last - __first), __layout_.__end_ptr(), __p)); } return __make_iter(__p); } @@ -1168,13 +1118,13 @@ vector<_Tp, _Allocator>::erase(const_iterator __first, const_iterator __last) { template _LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::__move_range(pointer __from_s, pointer __from_e, pointer __to) { - pointer __old_last = this->__end_; + pointer __old_last = __layout_.__end_ptr(); difference_type __n = __old_last - __to; { pointer __i = __from_s + __n; _ConstructTransaction __tx(*this, __from_e - __i); for (pointer __pos = __tx.__pos_; __i < __from_e; ++__i, (void)++__pos, __tx.__pos_ = __pos) { - __alloc_traits::construct(this->__alloc_, std::__to_address(__pos), std::move(*__i)); + __alloc_traits::construct(this->__layout_.__alloc(), std::__to_address(__pos), std::move(*__i)); } } std::move_backward(__from_s, __from_s + __n, __old_last); @@ -1183,21 +1133,22 @@ vector<_Tp, _Allocator>::__move_range(pointer __from_s, pointer __from_e, pointe template _LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::iterator vector<_Tp, _Allocator>::insert(const_iterator __position, const_reference __x) { - pointer __p = this->__begin_ + (__position - begin()); - if (this->__end_ < this->__cap_) { - if (__p == this->__end_) { + pointer __p = this->__layout_.__begin_ptr() + (__position - begin()); + if (!__layout_.__is_full()) { + pointer __end = __layout_.__end_ptr(); + if (__p == __end) { __emplace_back_assume_capacity(__x); } else { - __move_range(__p, this->__end_, __p + 1); + __move_range(__p, __end, __p + 1); const_pointer __xr = pointer_traits::pointer_to(__x); - if (std::__is_pointer_in_range(std::__to_address(__p), std::__to_address(__end_), std::addressof(__x))) + if (std::__is_pointer_in_range(std::__to_address(__p), std::__to_address(__end), std::addressof(__x))) ++__xr; *__p = *__xr; } } else { - _SplitBuffer __v(__recommend(size() + 1), __p - this->__begin_, this->__alloc_); + _SplitBuffer __v(__recommend(size() + 1), __p - this->__layout_.__begin_ptr(), this->__layout_.__alloc()); __v.emplace_back(__x); - __p = __swap_out_circular_buffer(__v, __p); + __p = __layout_.__relocate_with_pivot(__v, __p); } return __make_iter(__p); } @@ -1205,18 +1156,19 @@ vector<_Tp, _Allocator>::insert(const_iterator __position, const_reference __x) template _LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::iterator vector<_Tp, _Allocator>::insert(const_iterator __position, value_type&& __x) { - pointer __p = this->__begin_ + (__position - begin()); - if (this->__end_ < this->__cap_) { - if (__p == this->__end_) { + pointer __p = this->__layout_.__begin_ptr() + (__position - begin()); + if (!__layout_.__is_full()) { + pointer __end = __layout_.__end_ptr(); + if (__p == __end) { __emplace_back_assume_capacity(std::move(__x)); } else { - __move_range(__p, this->__end_, __p + 1); + __move_range(__p, __end, __p + 1); *__p = std::move(__x); } } else { - _SplitBuffer __v(__recommend(size() + 1), __p - this->__begin_, this->__alloc_); + _SplitBuffer __v(__recommend(size() + 1), __p - this->__layout_.__begin_ptr(), this->__layout_.__alloc()); __v.emplace_back(std::move(__x)); - __p = __swap_out_circular_buffer(__v, __p); + __p = __layout_.__relocate_with_pivot(__v, __p); } return __make_iter(__p); } @@ -1225,19 +1177,20 @@ template template _LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::iterator vector<_Tp, _Allocator>::emplace(const_iterator __position, _Args&&... __args) { - pointer __p = this->__begin_ + (__position - begin()); - if (this->__end_ < this->__cap_) { - if (__p == this->__end_) { + pointer __p = this->__layout_.__begin_ptr() + (__position - begin()); + if (!__layout_.__is_full()) { + pointer __end = __layout_.__end_ptr(); + if (__p == __end) { __emplace_back_assume_capacity(std::forward<_Args>(__args)...); } else { - __temp_value __tmp(this->__alloc_, std::forward<_Args>(__args)...); - __move_range(__p, this->__end_, __p + 1); + __temp_value __tmp(this->__layout_.__alloc(), std::forward<_Args>(__args)...); + __move_range(__p, __end, __p + 1); *__p = std::move(__tmp.get()); } } else { - _SplitBuffer __v(__recommend(size() + 1), __p - this->__begin_, this->__alloc_); + _SplitBuffer __v(__recommend(size() + 1), __p - this->__layout_.__begin_ptr(), this->__layout_.__alloc()); __v.emplace_back(std::forward<_Args>(__args)...); - __p = __swap_out_circular_buffer(__v, __p); + __p = __layout_.__relocate_with_pivot(__v, __p); } return __make_iter(__p); } @@ -1245,27 +1198,28 @@ vector<_Tp, _Allocator>::emplace(const_iterator __position, _Args&&... __args) { template _LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::iterator vector<_Tp, _Allocator>::insert(const_iterator __position, size_type __n, const_reference __x) { - pointer __p = this->__begin_ + (__position - begin()); + pointer __p = this->__layout_.__begin_ptr() + (__position - begin()); if (__n > 0) { - if (__n <= static_cast(this->__cap_ - this->__end_)) { + if (__n <= __layout_.__remaining_capacity()) { size_type __old_n = __n; - pointer __old_last = this->__end_; - if (__n > static_cast(this->__end_ - __p)) { - size_type __cx = __n - (this->__end_ - __p); + pointer __end = __layout_.__end_ptr(); + pointer __old_last = __end; + if (__n > static_cast(__end - __p)) { + size_type __cx = __n - (__end - __p); __construct_at_end(__cx, __x); __n -= __cx; } if (__n > 0) { __move_range(__p, __old_last, __p + __old_n); const_pointer __xr = pointer_traits::pointer_to(__x); - if (std::__is_pointer_in_range(std::__to_address(__p), std::__to_address(__end_), std::addressof(__x))) + if (std::__is_pointer_in_range(std::__to_address(__p), std::__to_address(__end), std::addressof(__x))) __xr += __old_n; std::fill_n(__p, __n, *__xr); } } else { - _SplitBuffer __v(__recommend(size() + __n), __p - this->__begin_, this->__alloc_); + _SplitBuffer __v(__recommend(size() + __n), __p - this->__layout_.__begin_ptr(), this->__layout_.__alloc()); __v.__construct_at_end(__n, __x); - __p = __swap_out_circular_buffer(__v, __p); + __p = __layout_.__relocate_with_pivot(__v, __p); } } return __make_iter(__p); @@ -1276,30 +1230,38 @@ template _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI typename vector<_Tp, _Allocator>::iterator vector<_Tp, _Allocator>::__insert_with_sentinel(const_iterator __position, _InputIterator __first, _Sentinel __last) { difference_type __off = __position - begin(); - pointer __p = this->__begin_ + __off; - pointer __old_last = this->__end_; - for (; this->__end_ != this->__cap_ && __first != __last; ++__first) + pointer __p = this->__layout_.__begin_ptr() + __off; + pointer __old_last = __layout_.__end_ptr(); + for (; !__layout_.__is_full() && __first != __last; ++__first) __emplace_back_assume_capacity(*__first); if (__first == __last) - (void)std::rotate(__p, __old_last, this->__end_); + (void)std::rotate(__p, __old_last, __layout_.__end_ptr()); else { - _SplitBuffer __v(__alloc_); - auto __guard = std::__make_exception_guard( - _AllocatorDestroyRangeReverse(__alloc_, __old_last, this->__end_)); + _SplitBuffer __v(__layout_.__alloc()); + pointer __end = __layout_.__end_ptr(); + auto __guard = std::__make_exception_guard( + _AllocatorDestroyRangeReverse(__layout_.__alloc(), __old_last, __end)); __v.__construct_at_end_with_sentinel(std::move(__first), std::move(__last)); _SplitBuffer __merged( - __recommend(size() + __v.size()), __off, __alloc_); // has `__off` positions available at the front + __recommend(size() + __v.size()), __off, __layout_.__alloc()); // has `__off` positions available at the front std::__uninitialized_allocator_relocate( - __alloc_, std::__to_address(__old_last), std::__to_address(this->__end_), std::__to_address(__merged.end())); - __guard.__complete(); // Release the guard once objects in [__old_last_, __end_) have been successfully relocated. - __merged.__set_sentinel(__merged.end() + (this->__end_ - __old_last)); - this->__end_ = __old_last; + __layout_.__alloc(), + std::__to_address(__old_last), + std::__to_address(__layout_.__end_ptr()), + std::__to_address(__merged.end())); + __guard.__complete(); // Release the guard once objects in [__old_last_, __layout_.__end_ptr()) have been + // successfully relocated. + __merged.__set_sentinel(__merged.end() + (__layout_.__end_ptr() - __old_last)); + __layout_.__set_bound_using_pointer(__old_last); std::__uninitialized_allocator_relocate( - __alloc_, std::__to_address(__v.begin()), std::__to_address(__v.end()), std::__to_address(__merged.end())); + __layout_.__alloc(), + std::__to_address(__v.begin()), + std::__to_address(__v.end()), + std::__to_address(__merged.end())); __merged.__set_sentinel(__merged.size() + __v.size()); __v.__set_sentinel(__v.begin()); - __p = __swap_out_circular_buffer(__merged, __p); + __p = __layout_.__relocate_with_pivot(__merged, __p); } return __make_iter(__p); } @@ -1309,16 +1271,17 @@ template _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI typename vector<_Tp, _Allocator>::iterator vector<_Tp, _Allocator>::__insert_with_size( const_iterator __position, _Iterator __first, _Sentinel __last, difference_type __n) { - pointer __p = this->__begin_ + (__position - begin()); + pointer __p = this->__layout_.__begin_ptr() + (__position - begin()); if (__n > 0) { - if (__n <= this->__cap_ - this->__end_) { - pointer __old_last = this->__end_; - difference_type __dx = this->__end_ - __p; + if (__n <= static_cast(__layout_.__remaining_capacity())) { + pointer __end = __layout_.__end_ptr(); + pointer __old_last = __end; + difference_type __dx = __end - __p; if (__n > __dx) { #if _LIBCPP_STD_VER >= 23 if constexpr (!forward_iterator<_Iterator>) { __construct_at_end(std::move(__first), std::move(__last), __n); - std::rotate(__p, __old_last, this->__end_); + std::rotate(__p, __old_last, __end); } else #endif { @@ -1334,9 +1297,9 @@ vector<_Tp, _Allocator>::__insert_with_size( __insert_assign_n_unchecked<_AlgPolicy>(std::move(__first), __n, __p); } } else { - _SplitBuffer __v(__recommend(size() + __n), __p - this->__begin_, this->__alloc_); + _SplitBuffer __v(__recommend(size() + __n), __p - this->__layout_.__begin_ptr(), this->__layout_.__alloc()); __v.__construct_at_end_with_size(std::move(__first), __n); - __p = __swap_out_circular_buffer(__v, __p); + __p = __layout_.__relocate_with_pivot(__v, __p); } } return __make_iter(__p); @@ -1349,12 +1312,12 @@ _LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::resize(size_type __n if (__new_size <= capacity()) { __construct_at_end(__new_size - __current_size); } else { - _SplitBuffer __v(__recommend(__new_size), __current_size, __alloc_); + _SplitBuffer __v(__recommend(__new_size), __current_size, __layout_.__alloc()); __v.__construct_at_end(__new_size - __current_size); - __swap_out_circular_buffer(__v); + __layout_.__relocate(__v); } } else if (__current_size > __new_size) { - this->__destruct_at_end(this->__begin_ + __new_size); + this->__destruct_at_end(this->__layout_.__begin_ptr() + __new_size); } } @@ -1365,12 +1328,12 @@ _LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::resize(size_type __n if (__new_size <= capacity()) __construct_at_end(__new_size - __current_size, __x); else { - _SplitBuffer __v(__recommend(__new_size), __current_size, __alloc_); + _SplitBuffer __v(__recommend(__new_size), __current_size, __layout_.__alloc()); __v.__construct_at_end(__new_size - __current_size, __x); - __swap_out_circular_buffer(__v); + __layout_.__relocate(__v); } } else if (__current_size > __new_size) { - this->__destruct_at_end(this->__begin_ + __new_size); + this->__destruct_at_end(this->__layout_.__begin_ptr() + __new_size); } } @@ -1383,29 +1346,10 @@ _LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::swap(vector& __x) #endif { _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR( - __alloc_traits::propagate_on_container_swap::value || this->__alloc_ == __x.__alloc_, + __alloc_traits::propagate_on_container_swap::value || __layout_.__alloc() == __x.__layout_.__alloc(), "vector::swap: Either propagate_on_container_swap must be true" " or the allocators must compare equal"); - std::swap(this->__begin_, __x.__begin_); - std::swap(this->__end_, __x.__end_); - std::swap(this->__cap_, __x.__cap_); - std::__swap_allocator(this->__alloc_, __x.__alloc_); -} - -template -_LIBCPP_CONSTEXPR_SINCE_CXX20 bool vector<_Tp, _Allocator>::__invariants() const { - if (this->__begin_ == nullptr) { - if (this->__end_ != nullptr || this->__cap_ != nullptr) - return false; - } else { - if (this->__begin_ > this->__end_) - return false; - if (this->__begin_ == this->__cap_) - return false; - if (this->__end_ > this->__cap_) - return false; - } - return true; + __layout_.__swap(__x.__layout_); } #if _LIBCPP_STD_VER >= 20 diff --git a/libcxx/include/module.modulemap.in b/libcxx/include/module.modulemap.in index b9ddfd5bbf4cd..b07d66a083c13 100644 --- a/libcxx/include/module.modulemap.in +++ b/libcxx/include/module.modulemap.in @@ -2258,6 +2258,7 @@ module std [system] { export std.format.formatter } + module layout { header "__vector/layout.h" } module pmr { header "__vector/pmr.h" diff --git a/libcxx/test/std/algorithms/alg.modifying.operations/alg.swap/ranges.swap_ranges.pass.cpp b/libcxx/test/std/algorithms/alg.modifying.operations/alg.swap/ranges.swap_ranges.pass.cpp index 85557ecbbfabc..f188a636d8254 100644 --- a/libcxx/test/std/algorithms/alg.modifying.operations/alg.swap/ranges.swap_ranges.pass.cpp +++ b/libcxx/test/std/algorithms/alg.modifying.operations/alg.swap/ranges.swap_ranges.pass.cpp @@ -9,6 +9,7 @@ // // UNSUPPORTED: c++03, c++11, c++14, c++17 +// ADDITIONAL_COMPILE_FLAGS(has-fconstexpr-steps): -fconstexpr-steps=9000000 // template S1, input_iterator I2, sentinel_for S2> // requires indirectly_swappable diff --git a/libcxx/utils/gdb/libcxx/printers.py b/libcxx/utils/gdb/libcxx/printers.py index 06867b9015555..ddd0f9b0f0add 100644 --- a/libcxx/utils/gdb/libcxx/printers.py +++ b/libcxx/utils/gdb/libcxx/printers.py @@ -367,17 +367,35 @@ def __init__(self, val): """Set val, length, capacity, and iterator for bool and normal vectors.""" self.val = val self.typename = _remove_generics(_prettify_typename(val.type)) - begin = self.val["__begin_"] if self.val.type.template_argument(0).code == gdb.TYPE_CODE_BOOL: self.typename += "" + begin = self.val["__begin_"] self.length = self.val["__size_"] bits_per_word = self.val["__bits_per_word"] self.capacity = self.val["__cap_"] * bits_per_word self.iterator = self._VectorBoolIterator(begin, self.length, bits_per_word) else: - end = self.val["__end_"] - self.length = end - begin - self.capacity = self.val["__cap_"] - begin + layout = self.val["__layout_"] + if layout: + fields = layout.type.fields() + begin = layout["__begin_"] + bound = layout[fields[1]] + + # We test for integers because `vector::size_type` is required to + # be an unsigned integer, whereas `vector::pointer` can be any + # type that satisfies the Cpp17NullablePointer requirements. + if bound.type.strip_typedefs().code == gdb.TYPE_CODE_INT: + self.length = layout["__size_"] + self.capacity = layout["__capacity_"] + else: + self.length = layout["__end_"] - begin + self.capacity = layout["__capacity_"] - begin + else: + begin = self.val["__begin_"] + self.length = self.val["__end_"] - begin + self.capacity = self.val["__cap_"] - begin + + end = begin + self.length self.iterator = self._VectorIterator(begin, end) def to_string(self):