Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

segment_map: Also segment the bucket array #1

Merged
merged 1 commit into from
Mar 19, 2024
Merged

segment_map: Also segment the bucket array #1

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
81 changes: 47 additions & 34 deletions include/ankerl/unordered_dense.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -810,7 +810,8 @@ class table : public std::conditional_t<is_map_v<T>, base_table_type_map<T>, bas
private:
using bucket_alloc =
typename std::allocator_traits<typename value_container_type::allocator_type>::template rebind_alloc<Bucket>;
using bucket_alloc_traits = std::allocator_traits<bucket_alloc>;
using underlying_bucket_type =
std::conditional_t<IsSegmented, segmented_vector<Bucket, bucket_alloc>, std::vector<Bucket, bucket_alloc>>;

static constexpr uint8_t initial_shifts = 64 - 2; // 2^(64-m_shift) number of buckets
static constexpr float default_max_load_factor = 0.8F;
Expand Down Expand Up @@ -839,24 +840,26 @@ class table : public std::conditional_t<is_map_v<T>, base_table_type_map<T>, bas
static_assert(std::is_trivially_copyable_v<Bucket>, "assert we can just memset / memcpy");

value_container_type m_values{}; // Contains all the key-value pairs in one densely stored container. No holes.
using bucket_pointer = typename std::allocator_traits<bucket_alloc>::pointer;
bucket_pointer m_buckets{};
size_t m_num_buckets = 0;
underlying_bucket_type m_buckets{};
size_t m_max_bucket_capacity = 0;
float m_max_load_factor = default_max_load_factor;
Hash m_hash{};
KeyEqual m_equal{};
uint8_t m_shifts = initial_shifts;

[[nodiscard]] auto next(value_idx_type bucket_idx) const -> value_idx_type {
return ANKERL_UNORDERED_DENSE_UNLIKELY(bucket_idx + 1U == m_num_buckets)
return ANKERL_UNORDERED_DENSE_UNLIKELY(bucket_idx + 1U == bucket_count())
? 0
: static_cast<value_idx_type>(bucket_idx + 1U);
}

// Helper to access bucket through pointer types
[[nodiscard]] static constexpr auto at(bucket_pointer bucket_ptr, size_t offset) -> Bucket& {
return *(bucket_ptr + static_cast<typename std::allocator_traits<bucket_alloc>::difference_type>(offset));
[[nodiscard]] static constexpr auto at(underlying_bucket_type& bucket, size_t offset) -> Bucket& {
return bucket[offset];
}

[[nodiscard]] static constexpr auto at(const underlying_bucket_type& bucket, size_t offset) -> const Bucket& {
return bucket[offset];
}

// use the dist_inc and dist_dec functions so that uint16_t types work without warning
Expand Down Expand Up @@ -946,7 +949,13 @@ class table : public std::conditional_t<is_map_v<T>, base_table_type_map<T>, bas
} else {
m_shifts = other.m_shifts;
allocate_buckets_from_shift();
std::memcpy(m_buckets, other.m_buckets, sizeof(Bucket) * bucket_count());
if constexpr (IsSegmented) {
for (auto i = 0UL; i < bucket_count(); ++i) {
at(m_buckets, i) = at(other.m_buckets, i);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Perhaps in the future this could still do the memcpy for each segment, right?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, though we'd need some form of fragment iterator API.

}
} else {
std::memcpy(m_buckets.data(), other.m_buckets.data(), sizeof(Bucket) * bucket_count());
}
}
}

Expand All @@ -958,30 +967,36 @@ class table : public std::conditional_t<is_map_v<T>, base_table_type_map<T>, bas
}

void deallocate_buckets() {
auto ba = bucket_alloc(m_values.get_allocator());
if (nullptr != m_buckets) {
bucket_alloc_traits::deallocate(ba, m_buckets, bucket_count());
m_buckets = nullptr;
}
m_num_buckets = 0;
m_buckets.clear();
m_buckets.shrink_to_fit();
m_max_bucket_capacity = 0;
}

void allocate_buckets_from_shift() {
auto ba = bucket_alloc(m_values.get_allocator());
m_num_buckets = calc_num_buckets(m_shifts);
m_buckets = bucket_alloc_traits::allocate(ba, m_num_buckets);
if (m_num_buckets == max_bucket_count()) {
auto num_buckets = calc_num_buckets(m_shifts);
if constexpr (IsSegmented) {
m_buckets.reserve(num_buckets);
for (size_t i = m_buckets.size(); i < num_buckets; ++i) {
m_buckets.emplace_back();
}
} else {
m_buckets.resize(num_buckets);
}
if (num_buckets == max_bucket_count()) {
// reached the maximum, make sure we can use each bucket
m_max_bucket_capacity = max_bucket_count();
} else {
m_max_bucket_capacity = static_cast<value_idx_type>(static_cast<float>(m_num_buckets) * max_load_factor());
m_max_bucket_capacity = static_cast<value_idx_type>(static_cast<float>(num_buckets) * max_load_factor());
}
}

void clear_buckets() {
if (m_buckets != nullptr) {
std::memset(&*m_buckets, 0, sizeof(Bucket) * bucket_count());
if constexpr (IsSegmented) {
for (auto&& e : m_buckets) {
std::memset(&e, 0, sizeof(e));
}
} else {
std::memset(m_buckets.data(), 0, sizeof(Bucket) * bucket_count());
}
}

Expand All @@ -1004,7 +1019,9 @@ class table : public std::conditional_t<is_map_v<T>, base_table_type_map<T>, bas
on_error_bucket_overflow();
}
--m_shifts;
deallocate_buckets();
if constexpr (!IsSegmented) {
deallocate_buckets();
}
allocate_buckets_from_shift();
clear_and_fill_buckets_from_values();
}
Expand Down Expand Up @@ -1178,6 +1195,7 @@ class table : public std::conditional_t<is_map_v<T>, base_table_type_map<T>, bas
KeyEqual const& equal = KeyEqual(),
allocator_type const& alloc_or_container = allocator_type())
: m_values(alloc_or_container)
, m_buckets(alloc_or_container)
, m_hash(hash)
, m_equal(equal) {
if (0 != bucket_count) {
Expand Down Expand Up @@ -1253,12 +1271,7 @@ class table : public std::conditional_t<is_map_v<T>, base_table_type_map<T>, bas
table(std::initializer_list<value_type> init, size_type bucket_count, Hash const& hash, allocator_type const& alloc)
: table(init, bucket_count, hash, KeyEqual(), alloc) {}

~table() {
if (nullptr != m_buckets) {
auto ba = bucket_alloc(m_values.get_allocator());
bucket_alloc_traits::deallocate(ba, m_buckets, bucket_count());
}
}
~table() {}

auto operator=(table const& other) -> table& {
if (&other != this) {
Expand All @@ -1283,8 +1296,8 @@ class table : public std::conditional_t<is_map_v<T>, base_table_type_map<T>, bas

// we can only reuse m_buckets when both maps have the same allocator!
if (get_allocator() == other.get_allocator()) {
m_buckets = std::exchange(other.m_buckets, nullptr);
m_num_buckets = std::exchange(other.m_num_buckets, 0);
m_buckets = std::move(other.m_buckets);
other.m_buckets.clear();
m_max_bucket_capacity = std::exchange(other.m_max_bucket_capacity, 0);
m_shifts = std::exchange(other.m_shifts, initial_shifts);
m_max_load_factor = std::exchange(other.m_max_load_factor, default_max_load_factor);
Expand Down Expand Up @@ -1421,7 +1434,7 @@ class table : public std::conditional_t<is_map_v<T>, base_table_type_map<T>, bas
on_error_too_many_elements();
}
auto shifts = calc_shifts_for_size(container.size());
if (0 == m_num_buckets || shifts < m_shifts || container.get_allocator() != m_values.get_allocator()) {
if (0 == bucket_count() || shifts < m_shifts || container.get_allocator() != m_values.get_allocator()) {
m_shifts = shifts;
deallocate_buckets();
allocate_buckets_from_shift();
Expand Down Expand Up @@ -1821,7 +1834,7 @@ class table : public std::conditional_t<is_map_v<T>, base_table_type_map<T>, bas
// bucket interface ///////////////////////////////////////////////////////

auto bucket_count() const noexcept -> size_t { // NOLINT(modernize-use-nodiscard)
return m_num_buckets;
return m_buckets.size();
}

static constexpr auto max_bucket_count() noexcept -> size_t { // NOLINT(modernize-use-nodiscard)
Expand All @@ -1840,7 +1853,7 @@ class table : public std::conditional_t<is_map_v<T>, base_table_type_map<T>, bas

void max_load_factor(float ml) {
m_max_load_factor = ml;
if (m_num_buckets != max_bucket_count()) {
if (bucket_count() != max_bucket_count()) {
m_max_bucket_capacity = static_cast<value_idx_type>(static_cast<float>(bucket_count()) * max_load_factor());
}
}
Expand All @@ -1864,7 +1877,7 @@ class table : public std::conditional_t<is_map_v<T>, base_table_type_map<T>, bas
m_values.reserve(capa);
}
auto shifts = calc_shifts_for_size((std::max)(capa, size()));
if (0 == m_num_buckets || shifts < m_shifts) {
if (0 == bucket_count() || shifts < m_shifts) {
m_shifts = shifts;
deallocate_buckets();
allocate_buckets_from_shift();
Expand Down