Skip to content

Commit

Permalink
Auto merge of #39109 - michaelwoerister:incr-comp-cache-cleanup, r=ni…
Browse files Browse the repository at this point in the history
…komatsakis

incr.comp.: Delete orphaned work-products.

The new partitioning scheme uncovered a hole in our incr. comp. cache directory garbage collection. So far, we relied on unneeded work products being deleted during the initial cache invalidation phase. However, we the new scheme, we get object files/work products that only contain code from upstream crates. Sometimes this code is not needed anymore (because all callers have been removed from the source) but because nothing that actually influences the contents of these work products had changed, we never deleted them from disk.

r? @nikomatsakis
  • Loading branch information
bors committed Jan 17, 2017
2 parents aedebfe + fe025d4 commit bd8e9b0
Show file tree
Hide file tree
Showing 6 changed files with 51 additions and 12 deletions.
6 changes: 6 additions & 0 deletions src/librustc/dep_graph/graph.rs
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,12 @@ impl DepGraph {
pub fn work_products(&self) -> Ref<FxHashMap<Arc<WorkProductId>, WorkProduct>> {
self.data.work_products.borrow()
}

/// Access the map of work-products created during the cached run. Only
/// used during saving of the dep-graph.
pub fn previous_work_products(&self) -> Ref<FxHashMap<Arc<WorkProductId>, WorkProduct>> {
self.data.previous_work_products.borrow()
}
}

/// A "work product" is an intermediate result that we save into the
Expand Down
1 change: 1 addition & 0 deletions src/librustc_incremental/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -57,3 +57,4 @@ pub use persist::save_trans_partition;
pub use persist::save_work_products;
pub use persist::in_incr_comp_dir;
pub use persist::finalize_session_directory;
pub use persist::delete_workproduct_files;
14 changes: 2 additions & 12 deletions src/librustc_incremental/persist/load.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ use rustc::ty::TyCtxt;
use rustc_data_structures::fx::{FxHashSet, FxHashMap};
use rustc_serialize::Decodable as RustcDecodable;
use rustc_serialize::opaque::Decoder;
use std::fs;
use std::path::{Path};

use IncrementalHashesMap;
Expand All @@ -29,6 +28,7 @@ use super::dirty_clean;
use super::hash::*;
use super::fs::*;
use super::file_format;
use super::work_product;

pub type DirtyNodes = FxHashSet<DepNode<DefPathIndex>>;

Expand Down Expand Up @@ -322,17 +322,7 @@ fn reconcile_work_products<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
fn delete_dirty_work_product(tcx: TyCtxt,
swp: SerializedWorkProduct) {
debug!("delete_dirty_work_product({:?})", swp);
for &(_, ref file_name) in &swp.work_product.saved_files {
let path = in_incr_comp_dir_sess(tcx.sess, file_name);
match fs::remove_file(&path) {
Ok(()) => { }
Err(err) => {
tcx.sess.warn(
&format!("file-system error deleting outdated file `{}`: {}",
path.display(), err));
}
}
}
work_product::delete_workproduct_files(tcx.sess, &swp.work_product);
}

fn load_prev_metadata_hashes(tcx: TyCtxt,
Expand Down
1 change: 1 addition & 0 deletions src/librustc_incremental/persist/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -29,3 +29,4 @@ pub use self::load::load_dep_graph;
pub use self::save::save_dep_graph;
pub use self::save::save_work_products;
pub use self::work_product::save_trans_partition;
pub use self::work_product::delete_workproduct_files;
26 changes: 26 additions & 0 deletions src/librustc_incremental/persist/save.rs
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ use super::preds::*;
use super::fs::*;
use super::dirty_clean;
use super::file_format;
use super::work_product;
use calculate_svh::IchHasher;

pub fn save_dep_graph<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
Expand Down Expand Up @@ -84,6 +85,31 @@ pub fn save_work_products(sess: &Session) {
let _ignore = sess.dep_graph.in_ignore();
let path = work_products_path(sess);
save_in(sess, path, |e| encode_work_products(sess, e));

// We also need to clean out old work-products, as not all of them are
// deleted during invalidation. Some object files don't change their
// content, they are just not needed anymore.
let new_work_products = sess.dep_graph.work_products();
let previous_work_products = sess.dep_graph.previous_work_products();

for (id, wp) in previous_work_products.iter() {
if !new_work_products.contains_key(id) {
work_product::delete_workproduct_files(sess, wp);
debug_assert!(wp.saved_files.iter().all(|&(_, ref file_name)| {
!in_incr_comp_dir_sess(sess, file_name).exists()
}));
}
}

// Check that we did not delete one of the current work-products:
debug_assert!({
new_work_products.iter()
.flat_map(|(_, wp)| wp.saved_files
.iter()
.map(|&(_, ref name)| name))
.map(|name| in_incr_comp_dir_sess(sess, name))
.all(|path| path.exists())
});
}

fn save_in<F>(sess: &Session, path_buf: PathBuf, encode: F)
Expand Down
15 changes: 15 additions & 0 deletions src/librustc_incremental/persist/work_product.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ use rustc::session::config::OutputType;
use rustc::util::fs::link_or_copy;
use std::path::PathBuf;
use std::sync::Arc;
use std::fs as std_fs;

pub fn save_trans_partition(sess: &Session,
cgu_name: &str,
Expand Down Expand Up @@ -61,3 +62,17 @@ pub fn save_trans_partition(sess: &Session,

sess.dep_graph.insert_work_product(&work_product_id, work_product);
}

pub fn delete_workproduct_files(sess: &Session, work_product: &WorkProduct) {
for &(_, ref file_name) in &work_product.saved_files {
let path = in_incr_comp_dir_sess(sess, file_name);
match std_fs::remove_file(&path) {
Ok(()) => { }
Err(err) => {
sess.warn(
&format!("file-system error deleting outdated file `{}`: {}",
path.display(), err));
}
}
}
}

0 comments on commit bd8e9b0

Please sign in to comment.