From 101fd7005c537e7ef2f2e80e47fe3f04621d48de Mon Sep 17 00:00:00 2001 From: Martin Leitner-Ankerl Date: Mon, 1 Aug 2022 21:50:29 +0200 Subject: [PATCH 1/2] for #16: customizable bucket types --- include/ankerl/unordered_dense.h | 128 ++++++++++++++++++++----------- test/bench/quick_overall_map.cpp | 21 +++++ test/meson.build | 1 + test/unit/bucket.cpp | 29 +++++++ test/unit/max.cpp | 4 +- 5 files changed, 135 insertions(+), 48 deletions(-) create mode 100644 test/unit/bucket.cpp diff --git a/include/ankerl/unordered_dense.h b/include/ankerl/unordered_dense.h index 029c4600..91485b1d 100644 --- a/include/ankerl/unordered_dense.h +++ b/include/ankerl/unordered_dense.h @@ -40,6 +40,12 @@ # define ANKERL_UNORDERED_DENSE_CPP_VERSION __cplusplus #endif +#if defined(__GNUC__) +# define ANKERL_UNORDERED_DENSE_PACK(decl) decl __attribute__((__packed__)) +#elif defined(_MSC_VER) +# define ANKERL_UNORDERED_DENSE_PACK(decl) __pragma(pack(push, 1)) decl __pragma(pack(pop)) +#endif + #if ANKERL_UNORDERED_DENSE_CPP_VERSION < 201703L # error ankerl::unordered_dense requires C++17 or higher #else @@ -291,6 +297,28 @@ ANKERL_UNORDERED_DENSE_HASH_STATICCAST(unsigned long long); # pragma GCC diagnostic pop # endif +// bucket_type ////////////////////////////////////////////////////////// + +namespace bucket_type { + +struct standard { + static constexpr uint32_t DIST_INC = 1U << 8U; // skip 1 byte fingerprint + static constexpr uint32_t FINGERPRINT_MASK = DIST_INC - 1; // mask for 1 byte of fingerprint + + uint32_t dist_and_fingerprint; // upper 3 byte: distance to original bucket. lower byte: fingerprint from hash + uint32_t value_idx; // index into the m_values vector. +}; + +ANKERL_UNORDERED_DENSE_PACK(struct big { + static constexpr uint32_t DIST_INC = 1U << 8U; // skip 1 byte fingerprint + static constexpr uint32_t FINGERPRINT_MASK = DIST_INC - 1; // mask for 1 byte of fingerprint + + uint32_t dist_and_fingerprint; // upper 3 byte: distance to original bucket. lower byte: fingerprint from hash + size_t value_idx; // index into the m_values vector. +}); + +} // namespace bucket_type + namespace detail { struct nonesuch {}; @@ -328,17 +356,15 @@ template + class Allocator, + class Bucket> class table { - struct Bucket; using ValueContainer = typename std::vector, Key, std::pair>, Allocator>; using BucketAlloc = typename std::allocator_traits::template rebind_alloc; using BucketAllocTraits = std::allocator_traits; - static constexpr uint32_t BUCKET_DIST_INC = 1U << 8U; // skip 1 byte fingerprint - static constexpr uint32_t BUCKET_FINGERPRINT_MASK = BUCKET_DIST_INC - 1; // mask for 1 byte of fingerprint - static constexpr uint8_t INITIAL_SHIFTS = 64 - 3; // 2^(64-m_shift) number of buckets + static constexpr uint8_t INITIAL_SHIFTS = 64 - 3; // 2^(64-m_shift) number of buckets static constexpr float DEFAULT_MAX_LOAD_FACTOR = 0.8F; public: @@ -357,19 +383,19 @@ class table { using const_pointer = typename value_container_type::const_pointer; using iterator = typename value_container_type::iterator; using const_iterator = typename value_container_type::const_iterator; + using bucket_type = Bucket; private: - struct Bucket { - uint32_t dist_and_fingerprint; // upper 3 byte: distance to original bucket. lower byte: fingerprint from hash - uint32_t value_idx; // index into the m_values vector. - }; + using value_idx_type = decltype(Bucket::value_idx); + using dist_and_fingerprint_type = decltype(Bucket::dist_and_fingerprint); + static_assert(std::is_trivially_destructible_v, "assert there's no need to call destructor / std::destroy"); static_assert(std::is_trivially_copyable_v, "assert we can just memset / memcpy"); value_container_type m_values{}; // Contains all the key-value pairs in one densely stored container. No holes. Bucket* m_buckets = nullptr; size_t m_num_buckets = 0; - uint32_t m_max_bucket_capacity = 0; + value_idx_type m_max_bucket_capacity = 0; float m_max_load_factor = DEFAULT_MAX_LOAD_FACTOR; Hash m_hash{}; KeyEqual m_equal{}; @@ -388,8 +414,8 @@ class table { } } - [[nodiscard]] constexpr auto dist_and_fingerprint_from_hash(uint64_t hash) const -> uint32_t { - return BUCKET_DIST_INC | (hash & BUCKET_FINGERPRINT_MASK); + [[nodiscard]] constexpr auto dist_and_fingerprint_from_hash(uint64_t hash) const -> dist_and_fingerprint_type { + return Bucket::DIST_INC | (hash & Bucket::FINGERPRINT_MASK); } [[nodiscard]] constexpr auto bucket_idx_from_hash(uint64_t hash) const -> size_t { @@ -405,13 +431,13 @@ class table { } template - [[nodiscard]] auto next_while_less(K const& key) const -> std::pair { + [[nodiscard]] auto next_while_less(K const& key) const -> std::pair { auto hash = mixed_hash(key); auto dist_and_fingerprint = dist_and_fingerprint_from_hash(hash); auto bucket_idx = bucket_idx_from_hash(hash); while (dist_and_fingerprint < m_buckets[bucket_idx].dist_and_fingerprint) { - dist_and_fingerprint += BUCKET_DIST_INC; + dist_and_fingerprint += Bucket::DIST_INC; bucket_idx = next(bucket_idx); } return {dist_and_fingerprint, bucket_idx}; @@ -420,7 +446,7 @@ class table { void place_and_shift_up(Bucket bucket, size_t place) { while (0 != m_buckets[place].dist_and_fingerprint) { bucket = std::exchange(m_buckets[place], bucket); - bucket.dist_and_fingerprint += BUCKET_DIST_INC; + bucket.dist_and_fingerprint += Bucket::DIST_INC; place = next(place); } m_buckets[place] = bucket; @@ -466,7 +492,7 @@ class table { auto bucket_alloc = BucketAlloc(m_values.get_allocator()); m_num_buckets = calc_num_buckets(m_shifts); m_buckets = BucketAllocTraits::allocate(bucket_alloc, m_num_buckets); - m_max_bucket_capacity = static_cast(static_cast(m_num_buckets) * max_load_factor()); + m_max_bucket_capacity = static_cast(static_cast(m_num_buckets) * max_load_factor()); } void clear_buckets() { @@ -477,7 +503,8 @@ class table { void clear_and_fill_buckets_from_values() { clear_buckets(); - for (uint32_t value_idx = 0, end_idx = static_cast(m_values.size()); value_idx < end_idx; ++value_idx) { + for (value_idx_type value_idx = 0, end_idx = static_cast(m_values.size()); value_idx < end_idx; + ++value_idx) { auto const& key = get_key(m_values[value_idx]); auto [dist_and_fingerprint, bucket] = next_while_less(key); @@ -498,8 +525,8 @@ class table { // shift down until either empty or an element with correct spot is found auto next_bucket_idx = next(bucket_idx); - while (m_buckets[next_bucket_idx].dist_and_fingerprint >= BUCKET_DIST_INC * 2) { - m_buckets[bucket_idx] = {m_buckets[next_bucket_idx].dist_and_fingerprint - BUCKET_DIST_INC, + while (m_buckets[next_bucket_idx].dist_and_fingerprint >= Bucket::DIST_INC * 2) { + m_buckets[bucket_idx] = {m_buckets[next_bucket_idx].dist_and_fingerprint - Bucket::DIST_INC, m_buckets[next_bucket_idx].value_idx}; bucket_idx = std::exchange(next_bucket_idx, next(next_bucket_idx)); } @@ -515,7 +542,7 @@ class table { auto mh = mixed_hash(get_key(val)); bucket_idx = bucket_idx_from_hash(mh); - auto const values_idx_back = static_cast(m_values.size() - 1); + auto const values_idx_back = static_cast(m_values.size() - 1); while (values_idx_back != m_buckets[bucket_idx].value_idx) { bucket_idx = next(bucket_idx); } @@ -534,7 +561,7 @@ class table { while (dist_and_fingerprint == m_buckets[bucket_idx].dist_and_fingerprint && !m_equal(key, get_key(m_values[m_buckets[bucket_idx].value_idx]))) { - dist_and_fingerprint += BUCKET_DIST_INC; + dist_and_fingerprint += Bucket::DIST_INC; bucket_idx = next(bucket_idx); } @@ -555,7 +582,7 @@ class table { } template - auto do_place_element(uint32_t dist_and_fingerprint, size_t bucket_idx, K&& key, Args&&... args) + auto do_place_element(dist_and_fingerprint_type dist_and_fingerprint, size_t bucket_idx, K&& key, Args&&... args) -> std::pair { // emplace the new value. If that throws an exception, no harm done; index is still in a valid state @@ -564,7 +591,7 @@ class table { std::forward_as_tuple(std::forward(args)...)); // place element and shift up until we find an empty spot - uint32_t value_idx = static_cast(m_values.size()) - 1; + auto value_idx = static_cast(m_values.size()) - 1; place_and_shift_up({dist_and_fingerprint, value_idx}, bucket_idx); return {begin() + static_cast(value_idx), true}; } @@ -588,7 +615,7 @@ class table { } else if (dist_and_fingerprint > bucket->dist_and_fingerprint) { return do_place_element(dist_and_fingerprint, bucket_idx, std::forward(key), std::forward(args)...); } - dist_and_fingerprint += BUCKET_DIST_INC; + dist_and_fingerprint += Bucket::DIST_INC; bucket_idx = next(bucket_idx); } } @@ -608,14 +635,14 @@ class table { if (dist_and_fingerprint == bucket->dist_and_fingerprint && m_equal(key, get_key(m_values[bucket->value_idx]))) { return begin() + static_cast(bucket->value_idx); } - dist_and_fingerprint += BUCKET_DIST_INC; + dist_and_fingerprint += Bucket::DIST_INC; bucket_idx = next(bucket_idx); bucket = m_buckets + bucket_idx; if (dist_and_fingerprint == bucket->dist_and_fingerprint && m_equal(key, get_key(m_values[bucket->value_idx]))) { return begin() + static_cast(bucket->value_idx); } - dist_and_fingerprint += BUCKET_DIST_INC; + dist_and_fingerprint += Bucket::DIST_INC; bucket_idx = next(bucket_idx); bucket = m_buckets + bucket_idx; @@ -627,7 +654,7 @@ class table { } else if (dist_and_fingerprint > bucket->dist_and_fingerprint) { return end(); } - dist_and_fingerprint += BUCKET_DIST_INC; + dist_and_fingerprint += Bucket::DIST_INC; bucket_idx = next(bucket_idx); bucket = m_buckets + bucket_idx; } @@ -801,8 +828,8 @@ class table { return m_values.size(); } - [[nodiscard]] auto max_size() const noexcept -> size_t { - return std::numeric_limits::max(); + [[nodiscard]] static constexpr auto max_size() noexcept -> size_t { + return std::numeric_limits::max(); } // modifiers ////////////////////////////////////////////////////////////// @@ -894,12 +921,12 @@ class table { m_values.pop_back(); // value was already there, so get rid of it return {begin() + static_cast(m_buckets[bucket_idx].value_idx), false}; } - dist_and_fingerprint += BUCKET_DIST_INC; + dist_and_fingerprint += Bucket::DIST_INC; bucket_idx = next(bucket_idx); } // value is new, place the bucket and shift up until we find an empty spot - uint32_t value_idx = static_cast(m_values.size()) - 1; + value_idx_type value_idx = static_cast(m_values.size()) - 1; place_and_shift_up({dist_and_fingerprint, value_idx}, bucket_idx); return {begin() + static_cast(value_idx), true}; @@ -934,7 +961,7 @@ class table { auto hash = mixed_hash(get_key(*it)); auto bucket_idx = bucket_idx_from_hash(hash); - auto const value_idx_to_remove = static_cast(it - cbegin()); + auto const value_idx_to_remove = static_cast(it - cbegin()); while (m_buckets[bucket_idx].value_idx != value_idx_to_remove) { bucket_idx = next(bucket_idx); } @@ -1075,8 +1102,8 @@ class table { return m_num_buckets; } - auto max_bucket_count() const noexcept -> size_t { // NOLINT(modernize-use-nodiscard) - return std::numeric_limits::max(); + static constexpr auto max_bucket_count() noexcept -> size_t { // NOLINT(modernize-use-nodiscard) + return std::numeric_limits::max(); } // hash policy //////////////////////////////////////////////////////////// @@ -1091,7 +1118,7 @@ class table { void max_load_factor(float ml) { m_max_load_factor = ml; - m_max_bucket_capacity = static_cast(static_cast(bucket_count()) * max_load_factor()); + m_max_bucket_capacity = static_cast(static_cast(bucket_count()) * max_load_factor()); } void rehash(size_t count) { @@ -1168,21 +1195,30 @@ template , class KeyEqual = std::equal_to, - class Allocator = std::allocator>> -using map = detail::table; + class Allocator = std::allocator>, + class Bucket = bucket_type::standard> +using map = detail::table; -template , class KeyEqual = std::equal_to, class Allocator = std::allocator> -using set = detail::table; +template , + class KeyEqual = std::equal_to, + class Allocator = std::allocator, + class Bucket = bucket_type::standard> +using set = detail::table; # if ANKERL_UNORDERED_DENSE_PMR namespace pmr { -template , class KeyEqual = std::equal_to> -using map = detail::table>>; +template , + class KeyEqual = std::equal_to, + class Bucket = bucket_type::standard> +using map = detail::table>, Bucket>; -template , class KeyEqual = std::equal_to> -using set = detail::table>; +template , class KeyEqual = std::equal_to, class Bucket = bucket_type::standard> +using set = detail::table, Bucket>; } // namespace pmr @@ -1199,9 +1235,9 @@ using set = detail::table -auto erase_if(ankerl::unordered_dense::detail::table& map, Pred pred) -> size_t { - using Map = ankerl::unordered_dense::detail::table; +template +auto erase_if(ankerl::unordered_dense::detail::table& map, Pred pred) -> size_t { + using Map = ankerl::unordered_dense::detail::table; // going back to front because erase() invalidates the end iterator auto const old_size = map.size(); diff --git a/test/bench/quick_overall_map.cpp b/test/bench/quick_overall_map.cpp index 681890da..d3d79f20 100644 --- a/test/bench/quick_overall_map.cpp +++ b/test/bench/quick_overall_map.cpp @@ -210,6 +210,27 @@ TEST_CASE("bench_quick_overall_udm" * doctest::test_suite("bench") * doctest::sk fmt::print("{} bench_quick_overall_map_udm\n", geomean1(bench)); } +TEST_CASE("bench_quick_overall_udm_bigbucket" * doctest::test_suite("bench") * doctest::skip()) { + ankerl::nanobench::Bench bench; + // bench.minEpochTime(1s); + benchAll, + std::equal_to, + std::allocator>, + ankerl::unordered_dense::bucket_type::big>>( + &bench, "ankerl::unordered_dense::map"); + + benchAll, + std::equal_to, + std::allocator>, + ankerl::unordered_dense::bucket_type::big>>( + &bench, "ankerl::unordered_dense::map"); + fmt::print("{} bench_quick_overall_map_udm\n", geomean1(bench)); +} + template void testBig() { Map map; diff --git a/test/meson.build b/test/meson.build index e143674e..93ad8a8e 100644 --- a/test/meson.build +++ b/test/meson.build @@ -15,6 +15,7 @@ test_sources = [ 'unit/assign_to_move.cpp', 'unit/assignment_combinations.cpp', 'unit/at.cpp', + 'unit/bucket.cpp', 'unit/contains.cpp', 'unit/copy_and_assign_maps.cpp', 'unit/copyassignment.cpp', diff --git a/test/unit/bucket.cpp b/test/unit/bucket.cpp new file mode 100644 index 00000000..0c0253e9 --- /dev/null +++ b/test/unit/bucket.cpp @@ -0,0 +1,29 @@ +#include + +#include + +#include +#include // for out_of_range + +using Map = ankerl::unordered_dense::map; + +// big bucket type allows 2^64 elements, but has more memory & CPU overhead. +using MapBig = ankerl::unordered_dense::map, + std::equal_to, + std::allocator>, + ankerl::unordered_dense::bucket_type::big>; + +static_assert(sizeof(Map::bucket_type) == 8U); +static_assert(sizeof(MapBig::bucket_type) == sizeof(size_t) + 4U); + +static_assert(Map::max_size() == std::numeric_limits::max()); +static_assert(MapBig::max_size() == std::numeric_limits::max()); + +static_assert(Map::max_bucket_count() == std::numeric_limits::max()); +static_assert(MapBig::max_bucket_count() == std::numeric_limits::max()); + +TEST_CASE("bucket") { + // TODO nothing here yet +} diff --git a/test/unit/max.cpp b/test/unit/max.cpp index c151bd86..ffc36165 100644 --- a/test/unit/max.cpp +++ b/test/unit/max.cpp @@ -8,12 +8,12 @@ TEST_CASE("max_size") { auto const map = ankerl::unordered_dense::map(); - REQUIRE(map.max_size() == std::numeric_limits::max()); + REQUIRE(map.max_size() == std::numeric_limits::max()); } TEST_CASE("max_bucket_count") { auto const map = ankerl::unordered_dense::map(); - REQUIRE(map.max_bucket_count() == std::numeric_limits::max()); + REQUIRE(map.max_bucket_count() == std::numeric_limits::max()); } TEST_CASE("max_load_factor") { From b0fbac446a38408de43eef10e3e02cd9e9a93aa5 Mon Sep 17 00:00:00 2001 From: Martin Leitner-Ankerl Date: Mon, 1 Aug 2022 21:54:42 +0200 Subject: [PATCH 2/2] add missing cast --- include/ankerl/unordered_dense.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/ankerl/unordered_dense.h b/include/ankerl/unordered_dense.h index 91485b1d..3f60cdc0 100644 --- a/include/ankerl/unordered_dense.h +++ b/include/ankerl/unordered_dense.h @@ -415,7 +415,7 @@ class table { } [[nodiscard]] constexpr auto dist_and_fingerprint_from_hash(uint64_t hash) const -> dist_and_fingerprint_type { - return Bucket::DIST_INC | (hash & Bucket::FINGERPRINT_MASK); + return Bucket::DIST_INC | (static_cast(hash) & Bucket::FINGERPRINT_MASK); } [[nodiscard]] constexpr auto bucket_idx_from_hash(uint64_t hash) const -> size_t {