From cb4a210655dbf5dc354a09b7185ca47e1e1925f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Thu, 15 Jul 2021 10:59:04 +0200 Subject: [PATCH 1/3] Optimize `find` for size --- src/raw/mod.rs | 43 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 35 insertions(+), 8 deletions(-) diff --git a/src/raw/mod.rs b/src/raw/mod.rs index 342193b5a3..6f15c19910 100644 --- a/src/raw/mod.rs +++ b/src/raw/mod.rs @@ -921,14 +921,13 @@ impl RawTable { /// Searches for an element in the table. #[inline] pub fn find(&self, hash: u64, mut eq: impl FnMut(&T) -> bool) -> Option> { - unsafe { - for bucket in self.iter_hash(hash) { - let elm = bucket.as_ref(); - if likely(eq(elm)) { - return Some(bucket); - } - } - None + let result = self.table.find_inner(hash, &mut |index| unsafe { + eq(self.bucket(index).as_ref()) + }); + + match result { + Some(index) => Some(unsafe { self.bucket(index) }), + None => None, } } @@ -1054,6 +1053,7 @@ impl RawTable { /// `RawIterHash`. Because we cannot make the `next` method unsafe on the /// `RawIterHash` struct, we have to make the `iter_hash` method unsafe. #[cfg_attr(feature = "inline-more", inline)] + #[allow(dead_code)] // Used when the `raw` API is enabled pub unsafe fn iter_hash(&self, hash: u64) -> RawIterHash<'_, T, A> { RawIterHash::new(self, hash) } @@ -1255,6 +1255,33 @@ impl RawTableInner { } } + /// Searches for an element in the table. + #[inline] + fn find_inner(&self, hash: u64, eq: &mut dyn FnMut(usize) -> bool) -> Option { + unsafe { + let h2_hash = h2(hash); + let mut probe_seq = self.probe_seq(hash); + + loop { + let group = Group::load(self.ctrl(probe_seq.pos)); + + for bit in group.match_byte(h2_hash) { + let index = (probe_seq.pos + bit) & self.bucket_mask; + + if likely(eq(index)) { + return Some(index); + } + } + + if likely(group.match_empty().any_bit_set()) { + return None; + } + + probe_seq.move_next(self.bucket_mask); + } + } + } + #[allow(clippy::mut_mut)] #[inline] unsafe fn prepare_rehash_in_place(&mut self) { From 681212fd0898cff4d16385af28a1a63ad494961f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Thu, 15 Jul 2021 18:14:04 +0200 Subject: [PATCH 2/3] Tweaks --- src/raw/mod.rs | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/raw/mod.rs b/src/raw/mod.rs index 6f15c19910..496ac1a4a8 100644 --- a/src/raw/mod.rs +++ b/src/raw/mod.rs @@ -925,6 +925,7 @@ impl RawTable { eq(self.bucket(index).as_ref()) }); + // Avoid `Option::map` because it bloats LLVM IR. match result { Some(index) => Some(unsafe { self.bucket(index) }), None => None, @@ -1255,30 +1256,29 @@ impl RawTableInner { } } - /// Searches for an element in the table. + /// Searches for an element in the table. This uses dynamic dispatch to reduce the amount of + /// code generated, but it is eliminated by LLVM optimizations. #[inline] fn find_inner(&self, hash: u64, eq: &mut dyn FnMut(usize) -> bool) -> Option { - unsafe { - let h2_hash = h2(hash); - let mut probe_seq = self.probe_seq(hash); - - loop { - let group = Group::load(self.ctrl(probe_seq.pos)); + let h2_hash = h2(hash); + let mut probe_seq = self.probe_seq(hash); - for bit in group.match_byte(h2_hash) { - let index = (probe_seq.pos + bit) & self.bucket_mask; + loop { + let group = unsafe { Group::load(self.ctrl(probe_seq.pos)) }; - if likely(eq(index)) { - return Some(index); - } - } + for bit in group.match_byte(h2_hash) { + let index = (probe_seq.pos + bit) & self.bucket_mask; - if likely(group.match_empty().any_bit_set()) { - return None; + if likely(eq(index)) { + return Some(index); } + } - probe_seq.move_next(self.bucket_mask); + if likely(group.match_empty().any_bit_set()) { + return None; } + + probe_seq.move_next(self.bucket_mask); } } From 0c528efe5919b1e55e0810fe9c02a12a9836bb72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Thu, 15 Jul 2021 18:15:52 +0200 Subject: [PATCH 3/3] Use cfg! --- src/raw/mod.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/raw/mod.rs b/src/raw/mod.rs index 496ac1a4a8..567d16c03d 100644 --- a/src/raw/mod.rs +++ b/src/raw/mod.rs @@ -1054,7 +1054,7 @@ impl RawTable { /// `RawIterHash`. Because we cannot make the `next` method unsafe on the /// `RawIterHash` struct, we have to make the `iter_hash` method unsafe. #[cfg_attr(feature = "inline-more", inline)] - #[allow(dead_code)] // Used when the `raw` API is enabled + #[cfg(feature = "raw")] pub unsafe fn iter_hash(&self, hash: u64) -> RawIterHash<'_, T, A> { RawIterHash::new(self, hash) } @@ -2214,6 +2214,7 @@ struct RawIterHashInner<'a, A: Allocator + Clone> { impl<'a, T, A: Allocator + Clone> RawIterHash<'a, T, A> { #[cfg_attr(feature = "inline-more", inline)] + #[cfg(feature = "raw")] fn new(table: &'a RawTable, hash: u64) -> Self { RawIterHash { inner: RawIterHashInner::new(&table.table, hash), @@ -2223,6 +2224,7 @@ impl<'a, T, A: Allocator + Clone> RawIterHash<'a, T, A> { } impl<'a, A: Allocator + Clone> RawIterHashInner<'a, A> { #[cfg_attr(feature = "inline-more", inline)] + #[cfg(feature = "raw")] fn new(table: &'a RawTableInner, hash: u64) -> Self { unsafe { let h2_hash = h2(hash);