diff --git a/src/raw/mod.rs b/src/raw/mod.rs index 342193b5a3..567d16c03d 100644 --- a/src/raw/mod.rs +++ b/src/raw/mod.rs @@ -921,14 +921,14 @@ impl RawTable { /// Searches for an element in the table. #[inline] pub fn find(&self, hash: u64, mut eq: impl FnMut(&T) -> bool) -> Option> { - unsafe { - for bucket in self.iter_hash(hash) { - let elm = bucket.as_ref(); - if likely(eq(elm)) { - return Some(bucket); - } - } - None + let result = self.table.find_inner(hash, &mut |index| unsafe { + eq(self.bucket(index).as_ref()) + }); + + // Avoid `Option::map` because it bloats LLVM IR. + match result { + Some(index) => Some(unsafe { self.bucket(index) }), + None => None, } } @@ -1054,6 +1054,7 @@ impl RawTable { /// `RawIterHash`. Because we cannot make the `next` method unsafe on the /// `RawIterHash` struct, we have to make the `iter_hash` method unsafe. #[cfg_attr(feature = "inline-more", inline)] + #[cfg(feature = "raw")] pub unsafe fn iter_hash(&self, hash: u64) -> RawIterHash<'_, T, A> { RawIterHash::new(self, hash) } @@ -1255,6 +1256,32 @@ impl RawTableInner { } } + /// Searches for an element in the table. This uses dynamic dispatch to reduce the amount of + /// code generated, but it is eliminated by LLVM optimizations. + #[inline] + fn find_inner(&self, hash: u64, eq: &mut dyn FnMut(usize) -> bool) -> Option { + let h2_hash = h2(hash); + let mut probe_seq = self.probe_seq(hash); + + loop { + let group = unsafe { Group::load(self.ctrl(probe_seq.pos)) }; + + for bit in group.match_byte(h2_hash) { + let index = (probe_seq.pos + bit) & self.bucket_mask; + + if likely(eq(index)) { + return Some(index); + } + } + + if likely(group.match_empty().any_bit_set()) { + return None; + } + + probe_seq.move_next(self.bucket_mask); + } + } + #[allow(clippy::mut_mut)] #[inline] unsafe fn prepare_rehash_in_place(&mut self) { @@ -2187,6 +2214,7 @@ struct RawIterHashInner<'a, A: Allocator + Clone> { impl<'a, T, A: Allocator + Clone> RawIterHash<'a, T, A> { #[cfg_attr(feature = "inline-more", inline)] + #[cfg(feature = "raw")] fn new(table: &'a RawTable, hash: u64) -> Self { RawIterHash { inner: RawIterHashInner::new(&table.table, hash), @@ -2196,6 +2224,7 @@ impl<'a, T, A: Allocator + Clone> RawIterHash<'a, T, A> { } impl<'a, A: Allocator + Clone> RawIterHashInner<'a, A> { #[cfg_attr(feature = "inline-more", inline)] + #[cfg(feature = "raw")] fn new(table: &'a RawTableInner, hash: u64) -> Self { unsafe { let h2_hash = h2(hash);