Skip to content

Commit

Permalink
Rollup merge of rust-lang#65472 - Zoxc:sharded-dep-graph-2, r=nikomat…
Browse files Browse the repository at this point in the history
…sakis

Use a sharded dep node to dep node index map

Split out from rust-lang#61845 and based on rust-lang#63756.

r? @nikomatsakis
  • Loading branch information
tmandry authored Oct 18, 2019
2 parents 44acb1d + 42c0236 commit 809b3f0
Show file tree
Hide file tree
Showing 2 changed files with 44 additions and 15 deletions.
29 changes: 21 additions & 8 deletions src/librustc/dep_graph/graph.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ use rustc_data_structures::fx::{FxHashMap, FxHashSet};
use rustc_index::vec::{Idx, IndexVec};
use smallvec::SmallVec;
use rustc_data_structures::sync::{Lrc, Lock, AtomicU32, AtomicU64, Ordering};
use rustc_data_structures::sharded::{self, Sharded};
use std::sync::atomic::Ordering::SeqCst;
use std::env;
use std::hash::Hash;
Expand Down Expand Up @@ -381,7 +382,7 @@ impl DepGraph {
#[inline]
pub fn read(&self, v: DepNode) {
if let Some(ref data) = self.data {
let map = data.current.node_to_node_index.lock();
let map = data.current.node_to_node_index.get_shard_by_value(&v).lock();
if let Some(dep_node_index) = map.get(&v).copied() {
std::mem::drop(map);
data.read_index(dep_node_index);
Expand All @@ -405,6 +406,7 @@ impl DepGraph {
.unwrap()
.current
.node_to_node_index
.get_shard_by_value(dep_node)
.lock()
.get(dep_node)
.cloned()
Expand All @@ -414,7 +416,11 @@ impl DepGraph {
#[inline]
pub fn dep_node_exists(&self, dep_node: &DepNode) -> bool {
if let Some(ref data) = self.data {
data.current.node_to_node_index.lock().contains_key(dep_node)
data.current
.node_to_node_index
.get_shard_by_value(&dep_node)
.lock()
.contains_key(dep_node)
} else {
false
}
Expand Down Expand Up @@ -595,7 +601,11 @@ impl DepGraph {

#[cfg(not(parallel_compiler))]
{
debug_assert!(!data.current.node_to_node_index.lock().contains_key(dep_node));
debug_assert!(!data.current
.node_to_node_index
.get_shard_by_value(dep_node)
.lock()
.contains_key(dep_node));
debug_assert!(data.colors.get(prev_dep_node_index).is_none());
}

Expand Down Expand Up @@ -927,7 +937,7 @@ struct DepNodeData {
/// acquire the lock on `data.`
pub(super) struct CurrentDepGraph {
data: Lock<IndexVec<DepNodeIndex, DepNodeData>>,
node_to_node_index: Lock<FxHashMap<DepNode, DepNodeIndex>>,
node_to_node_index: Sharded<FxHashMap<DepNode, DepNodeIndex>>,

/// Used to trap when a specific edge is added to the graph.
/// This is used for debug purposes and is only active with `debug_assertions`.
Expand Down Expand Up @@ -985,8 +995,8 @@ impl CurrentDepGraph {

CurrentDepGraph {
data: Lock::new(IndexVec::with_capacity(new_node_count_estimate)),
node_to_node_index: Lock::new(FxHashMap::with_capacity_and_hasher(
new_node_count_estimate,
node_to_node_index: Sharded::new(|| FxHashMap::with_capacity_and_hasher(
new_node_count_estimate / sharded::SHARDS,
Default::default(),
)),
anon_id_seed: stable_hasher.finish(),
Expand Down Expand Up @@ -1035,7 +1045,10 @@ impl CurrentDepGraph {
edges: SmallVec<[DepNodeIndex; 8]>,
fingerprint: Fingerprint
) -> DepNodeIndex {
debug_assert!(!self.node_to_node_index.lock().contains_key(&dep_node));
debug_assert!(!self.node_to_node_index
.get_shard_by_value(&dep_node)
.lock()
.contains_key(&dep_node));
self.intern_node(dep_node, edges, fingerprint)
}

Expand All @@ -1045,7 +1058,7 @@ impl CurrentDepGraph {
edges: SmallVec<[DepNodeIndex; 8]>,
fingerprint: Fingerprint
) -> DepNodeIndex {
match self.node_to_node_index.lock().entry(dep_node) {
match self.node_to_node_index.get_shard_by_value(&dep_node).lock().entry(dep_node) {
Entry::Occupied(entry) => *entry.get(),
Entry::Vacant(entry) => {
let mut data = self.data.lock();
Expand Down
30 changes: 23 additions & 7 deletions src/librustc_data_structures/sharded.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ use std::hash::{Hasher, Hash};
use std::mem;
use std::borrow::Borrow;
use std::collections::hash_map::RawEntryMut;
use smallvec::SmallVec;
use crate::fx::{FxHasher, FxHashMap};
use crate::sync::{Lock, LockGuard};

Expand All @@ -18,7 +19,7 @@ const SHARD_BITS: usize = 5;
#[cfg(not(parallel_compiler))]
const SHARD_BITS: usize = 0;

const SHARDS: usize = 1 << SHARD_BITS;
pub const SHARDS: usize = 1 << SHARD_BITS;

/// An array of cache-line aligned inner locked structures with convenience methods.
#[derive(Clone)]
Expand All @@ -29,21 +30,36 @@ pub struct Sharded<T> {
impl<T: Default> Default for Sharded<T> {
#[inline]
fn default() -> Self {
Self::new(|| T::default())
}
}

impl<T> Sharded<T> {
#[inline]
pub fn new(mut value: impl FnMut() -> T) -> Self {
// Create a vector of the values we want
let mut values: SmallVec<[_; SHARDS]> = (0..SHARDS).map(|_| {
CacheAligned(Lock::new(value()))
}).collect();

// Create an unintialized array
let mut shards: mem::MaybeUninit<[CacheAligned<Lock<T>>; SHARDS]> =
mem::MaybeUninit::uninit();
let first = shards.as_mut_ptr() as *mut CacheAligned<Lock<T>>;

unsafe {
for i in 0..SHARDS {
first.add(i).write(CacheAligned(Lock::new(T::default())));
}
// Copy the values into our array
let first = shards.as_mut_ptr() as *mut CacheAligned<Lock<T>>;
values.as_ptr().copy_to_nonoverlapping(first, SHARDS);

// Ignore the content of the vector
values.set_len(0);

Sharded {
shards: shards.assume_init(),
}
}
}
}

impl<T> Sharded<T> {
#[inline]
pub fn get_shard_by_value<K: Hash + ?Sized>(&self, val: &K) -> &Lock<T> {
if SHARDS == 1 {
Expand Down

0 comments on commit 809b3f0

Please sign in to comment.