Skip to content

Commit

Permalink
Drop ahash dependency in favor of core's SipHasher
Browse files Browse the repository at this point in the history
tkaitchuck/aHash#196 bumped the MSRV of
`ahash` in a patch release, which makes it rather difficult for us
to have it as a dependency.

Further, it seems that `ahash` hasn't been particularly robust in
the past, notably
tkaitchuck/aHash#163 and
tkaitchuck/aHash#166.

Luckily, `core` provides `SipHasher` even on no-std (sadly its
SipHash-2-4 unlike the SipHash-1-3 used by the `DefaultHasher` in
`std`). Thus, we drop the `ahash` dependency entirely here and
simply wrap `SipHasher` for our `no-std` HashMaps.
  • Loading branch information
TheBlueMatt committed Feb 12, 2024
1 parent 6181758 commit 8ef043d
Show file tree
Hide file tree
Showing 6 changed files with 173 additions and 116 deletions.
2 changes: 1 addition & 1 deletion bench/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ name = "bench"
harness = false

[features]
hashbrown = ["lightning/hashbrown", "lightning/ahash"]
hashbrown = ["lightning/hashbrown"]

[dependencies]
lightning = { path = "../lightning", features = ["_test_utils", "criterion"] }
Expand Down
2 changes: 1 addition & 1 deletion ci/check-cfg-flags.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ def check_feature(feature):
pass
elif feature == "no-std":
pass
elif feature == "ahash":
elif feature == "possiblyrandom":
pass
elif feature == "hashbrown":
pass
Expand Down
14 changes: 2 additions & 12 deletions lightning/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ unsafe_revoked_tx_signing = []
# Override signing to not include randomness when generating signatures for test vectors.
_test_vectors = []

no-std = ["hashbrown", "ahash", "bitcoin/no-std", "core2/alloc", "libm"]
no-std = ["hashbrown", "possiblyrandom", "bitcoin/no-std", "core2/alloc", "libm"]
std = ["bitcoin/std"]

# Generates low-r bitcoin signatures, which saves 1 byte in 50% of the cases
Expand All @@ -43,24 +43,14 @@ default = ["std", "grind_signatures"]
bitcoin = { version = "0.30.2", default-features = false, features = ["secp-recovery"] }

hashbrown = { version = "0.13", optional = true }
ahash = { version = "0.8", optional = true, default-features = false }
possiblyrandom = { version = "0.1", optional = true, default-features = false }
hex = { package = "hex-conservative", version = "0.1.1", default-features = false }
regex = { version = "1.5.6", optional = true }
backtrace = { version = "0.3", optional = true }

core2 = { version = "0.3.0", optional = true, default-features = false }
libm = { version = "0.2", optional = true, default-features = false }

# Because ahash no longer (kinda poorly) does it for us, (roughly) list out the targets that
# getrandom supports and turn on ahash's `runtime-rng` feature for them.
[target.'cfg(not(any(target_os = "unknown", target_os = "none")))'.dependencies]
ahash = { version = "0.8", optional = true, default-features = false, features = ["runtime-rng"] }

# Not sure what target_os gets set to for sgx, so to be safe always enable runtime-rng for x86_64
# platforms (assuming LDK isn't being used on embedded x86-64 running directly on metal).
[target.'cfg(target_arch = "x86_64")'.dependencies]
ahash = { version = "0.8", optional = true, default-features = false, features = ["runtime-rng"] }

[dev-dependencies]
regex = "1.5.6"

Expand Down
103 changes: 1 addition & 102 deletions lightning/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -165,113 +165,12 @@ mod io_extras {
}

mod prelude {
#[cfg(feature = "hashbrown")]
extern crate hashbrown;
#[cfg(feature = "ahash")]
extern crate ahash;

pub use alloc::{vec, vec::Vec, string::String, collections::VecDeque, boxed::Box};

pub use alloc::borrow::ToOwned;
pub use alloc::string::ToString;

// For no-std builds, we need to use hashbrown, however, by default, it doesn't randomize the
// hashing and is vulnerable to HashDoS attacks. Thus, when not fuzzing, we use its default
// ahash hashing algorithm but randomize, opting to not randomize when fuzzing to avoid false
// positive branch coverage.

#[cfg(not(feature = "hashbrown"))]
mod std_hashtables {
pub(crate) use std::collections::{HashMap, HashSet, hash_map};

pub(crate) type OccupiedHashMapEntry<'a, K, V> =
std::collections::hash_map::OccupiedEntry<'a, K, V>;
pub(crate) type VacantHashMapEntry<'a, K, V> =
std::collections::hash_map::VacantEntry<'a, K, V>;
}
#[cfg(not(feature = "hashbrown"))]
pub(crate) use std_hashtables::*;

#[cfg(feature = "hashbrown")]
pub(crate) use self::hashbrown::hash_map;

#[cfg(all(feature = "hashbrown", fuzzing))]
mod nonrandomized_hashbrown {
pub(crate) use hashbrown::{HashMap, HashSet};

pub(crate) type OccupiedHashMapEntry<'a, K, V> =
hashbrown::hash_map::OccupiedEntry<'a, K, V, hashbrown::hash_map::DefaultHashBuilder>;
pub(crate) type VacantHashMapEntry<'a, K, V> =
hashbrown::hash_map::VacantEntry<'a, K, V, hashbrown::hash_map::DefaultHashBuilder>;
}
#[cfg(all(feature = "hashbrown", fuzzing))]
pub(crate) use nonrandomized_hashbrown::*;


#[cfg(all(feature = "hashbrown", not(fuzzing)))]
mod randomized_hashtables {
use super::*;
use ahash::RandomState;

pub(crate) type HashMap<K, V> = hashbrown::HashMap<K, V, RandomState>;
pub(crate) type HashSet<K> = hashbrown::HashSet<K, RandomState>;

pub(crate) type OccupiedHashMapEntry<'a, K, V> =
hashbrown::hash_map::OccupiedEntry<'a, K, V, RandomState>;
pub(crate) type VacantHashMapEntry<'a, K, V> =
hashbrown::hash_map::VacantEntry<'a, K, V, RandomState>;

pub(crate) fn new_hash_map<K, V>() -> HashMap<K, V> {
HashMap::with_hasher(RandomState::new())
}
pub(crate) fn hash_map_with_capacity<K, V>(cap: usize) -> HashMap<K, V> {
HashMap::with_capacity_and_hasher(cap, RandomState::new())
}
pub(crate) fn hash_map_from_iter<K: core::hash::Hash + Eq, V, I: IntoIterator<Item=(K, V)>>(iter: I) -> HashMap<K, V> {
let iter = iter.into_iter();
let min_size = iter.size_hint().0;
let mut res = HashMap::with_capacity_and_hasher(min_size, RandomState::new());
res.extend(iter);
res
}

pub(crate) fn new_hash_set<K>() -> HashSet<K> {
HashSet::with_hasher(RandomState::new())
}
pub(crate) fn hash_set_with_capacity<K>(cap: usize) -> HashSet<K> {
HashSet::with_capacity_and_hasher(cap, RandomState::new())
}
pub(crate) fn hash_set_from_iter<K: core::hash::Hash + Eq, I: IntoIterator<Item=K>>(iter: I) -> HashSet<K> {
let iter = iter.into_iter();
let min_size = iter.size_hint().0;
let mut res = HashSet::with_capacity_and_hasher(min_size, RandomState::new());
res.extend(iter);
res
}
}

#[cfg(any(not(feature = "hashbrown"), fuzzing))]
mod randomized_hashtables {
use super::*;

pub(crate) fn new_hash_map<K, V>() -> HashMap<K, V> { HashMap::new() }
pub(crate) fn hash_map_with_capacity<K, V>(cap: usize) -> HashMap<K, V> {
HashMap::with_capacity(cap)
}
pub(crate) fn hash_map_from_iter<K: core::hash::Hash + Eq, V, I: IntoIterator<Item=(K, V)>>(iter: I) -> HashMap<K, V> {
HashMap::from_iter(iter)
}

pub(crate) fn new_hash_set<K>() -> HashSet<K> { HashSet::new() }
pub(crate) fn hash_set_with_capacity<K>(cap: usize) -> HashSet<K> {
HashSet::with_capacity(cap)
}
pub(crate) fn hash_set_from_iter<K: core::hash::Hash + Eq, I: IntoIterator<Item=K>>(iter: I) -> HashSet<K> {
HashSet::from_iter(iter)
}
}

pub(crate) use randomized_hashtables::*;
pub(crate) use crate::util::hash_tables::*;
}

#[cfg(all(not(ldk_bench), feature = "backtrace", feature = "std", test))]
Expand Down
167 changes: 167 additions & 0 deletions lightning/src/util/hash_tables.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,167 @@
//! Generally LDK uses `std`'s `HashMap`s, however when building for no-std, LDK uses `hashbrown`'s
//! `HashMap`s with the `std` `SipHasher` and uses `getrandom` to opportunistically randomize it,
//! if randomization is available.
//!
//! This module simply re-exports the `HashMap` used in LDK for public consumption.

#[cfg(feature = "hashbrown")]
extern crate hashbrown;
#[cfg(feature = "possiblyrandom")]
extern crate possiblyrandom;

// For no-std builds, we need to use hashbrown, however, by default, it doesn't randomize the
// hashing and is vulnerable to HashDoS attacks. Thus, we use the core SipHasher when not using
// std, but use `getrandom` to randomize it if its available.

#[cfg(not(feature = "hashbrown"))]
mod std_hashtables {
pub use std::collections::HashMap;
pub use std::collections::hash_map::RandomState;

pub(crate) use std::collections::{HashSet, hash_map};

pub(crate) type OccupiedHashMapEntry<'a, K, V> =
std::collections::hash_map::OccupiedEntry<'a, K, V>;
pub(crate) type VacantHashMapEntry<'a, K, V> =
std::collections::hash_map::VacantEntry<'a, K, V>;
}
#[cfg(not(feature = "hashbrown"))]
pub use std_hashtables::*;

#[cfg(feature = "hashbrown")]
pub(crate) use self::hashbrown::hash_map;

#[cfg(all(feature = "hashbrown", fuzzing))]
mod nonrandomized_hashbrown {
pub use hashbrown::HashMap;
pub use std::collections::hash_map::RandomState;

pub(crate) use hashbrown::HashSet;

pub(crate) type OccupiedHashMapEntry<'a, K, V> =
hashbrown::hash_map::OccupiedEntry<'a, K, V, hashbrown::hash_map::DefaultHashBuilder>;
pub(crate) type VacantHashMapEntry<'a, K, V> =
hashbrown::hash_map::VacantEntry<'a, K, V, hashbrown::hash_map::DefaultHashBuilder>;
}
#[cfg(all(feature = "hashbrown", fuzzing))]
pub use nonrandomized_hashbrown::*;

#[cfg(all(feature = "hashbrown", not(fuzzing)))]
mod randomized_hashtables {
#[cfg(feature = "std")]
mod hasher {
pub use std::collections::hash_map::RandomState;
}
#[cfg(not(feature = "std"))]
mod hasher {
#![allow(deprecated)] // hash::SipHasher was deprecated in favor of something only in std.
use core::hash::{BuildHasher, SipHasher};

#[derive(Clone, Copy)]
/// A simple implementation of [`BuildHasher`] that uses `getrandom` to opportunistically
/// randomize, if the platform supports it.
pub struct RandomState {
k0: u64, k1: u64,
}

impl RandomState {
/// Constructs a new [`RandomState`] which may or may not be random, depending on the
/// target platform.
pub fn new() -> RandomState {
let (k0, k1);
#[cfg(feature = "possiblyrandom")] {
let mut keys = [0; 16];
possiblyrandom::getpossiblyrandom(&mut keys);

let mut k0_bytes = [0; 8];
let mut k1_bytes = [0; 8];
k0_bytes.copy_from_slice(&keys[..8]);
k1_bytes.copy_from_slice(&keys[8..]);
k0 = u64::from_le_bytes(k0_bytes);
k1 = u64::from_le_bytes(k1_bytes);
}
#[cfg(not(feature = "possiblyrandom"))] {
k0 = 0;
k1 = 0;
}
RandomState { k0, k1 }
}
}

impl Default for RandomState {
fn default() -> RandomState { RandomState::new() }
}

impl BuildHasher for RandomState {
type Hasher = SipHasher;
fn build_hasher(&self) -> SipHasher {
SipHasher::new_with_keys(self.k0, self.k1)
}
}
}

pub use hasher::*;
use super::*;

/// The HashMap type used in LDK.
pub type HashMap<K, V> = hashbrown::HashMap<K, V, RandomState>;
pub(crate) type HashSet<K> = hashbrown::HashSet<K, RandomState>;

pub(crate) type OccupiedHashMapEntry<'a, K, V> =
hashbrown::hash_map::OccupiedEntry<'a, K, V, RandomState>;
pub(crate) type VacantHashMapEntry<'a, K, V> =
hashbrown::hash_map::VacantEntry<'a, K, V, RandomState>;

pub(crate) fn new_hash_map<K, V>() -> HashMap<K, V> {
HashMap::with_hasher(RandomState::new())
}
pub(crate) fn hash_map_with_capacity<K, V>(cap: usize) -> HashMap<K, V> {
HashMap::with_capacity_and_hasher(cap, RandomState::new())
}
pub(crate) fn hash_map_from_iter<K: core::hash::Hash + Eq, V, I: IntoIterator<Item=(K, V)>>(iter: I) -> HashMap<K, V> {
let iter = iter.into_iter();
let min_size = iter.size_hint().0;
let mut res = HashMap::with_capacity_and_hasher(min_size, RandomState::new());
res.extend(iter);
res
}

pub(crate) fn new_hash_set<K>() -> HashSet<K> {
HashSet::with_hasher(RandomState::new())
}
pub(crate) fn hash_set_with_capacity<K>(cap: usize) -> HashSet<K> {
HashSet::with_capacity_and_hasher(cap, RandomState::new())
}
pub(crate) fn hash_set_from_iter<K: core::hash::Hash + Eq, I: IntoIterator<Item=K>>(iter: I) -> HashSet<K> {
let iter = iter.into_iter();
let min_size = iter.size_hint().0;
let mut res = HashSet::with_capacity_and_hasher(min_size, RandomState::new());
res.extend(iter);
res
}
}
#[cfg(all(feature = "hashbrown", not(fuzzing)))]
pub use randomized_hashtables::*;

#[cfg(any(not(feature = "hashbrown"), fuzzing))]
mod hashtable_constructors {
use super::*;

pub(crate) fn new_hash_map<K, V>() -> HashMap<K, V> { HashMap::new() }
pub(crate) fn hash_map_with_capacity<K, V>(cap: usize) -> HashMap<K, V> {
HashMap::with_capacity(cap)
}
pub(crate) fn hash_map_from_iter<K: core::hash::Hash + Eq, V, I: IntoIterator<Item=(K, V)>>(iter: I) -> HashMap<K, V> {
HashMap::from_iter(iter)
}

pub(crate) fn new_hash_set<K>() -> HashSet<K> { HashSet::new() }
pub(crate) fn hash_set_with_capacity<K>(cap: usize) -> HashSet<K> {
HashSet::with_capacity(cap)
}
pub(crate) fn hash_set_from_iter<K: core::hash::Hash + Eq, I: IntoIterator<Item=K>>(iter: I) -> HashSet<K> {
HashSet::from_iter(iter)
}
}
#[cfg(any(not(feature = "hashbrown"), fuzzing))]
pub(crate) use hashtable_constructors::*;
1 change: 1 addition & 0 deletions lightning/src/util/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ pub(crate) mod atomic_counter;
pub(crate) mod byte_utils;
pub(crate) mod transaction_utils;
pub(crate) mod time;
pub mod hash_tables;

pub mod indexed_map;

Expand Down

0 comments on commit 8ef043d

Please sign in to comment.