From 2e587df6e2215b9dd053879d2ad42044781fc1c4 Mon Sep 17 00:00:00 2001 From: Michael Woerister Date: Fri, 17 Aug 2018 15:25:46 +0200 Subject: [PATCH 1/7] Provide a way of accessing the ThinLTO module import map in rustc. --- src/librustc_codegen_llvm/llvm/ffi.rs | 9 +++++++++ src/rustllvm/PassWrapper.cpp | 27 +++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/src/librustc_codegen_llvm/llvm/ffi.rs b/src/librustc_codegen_llvm/llvm/ffi.rs index 51b0299e63f46..6c2601bf1ef12 100644 --- a/src/librustc_codegen_llvm/llvm/ffi.rs +++ b/src/librustc_codegen_llvm/llvm/ffi.rs @@ -363,6 +363,10 @@ extern { pub type ThinLTOData; } /// LLVMRustThinLTOBuffer extern { pub type ThinLTOBuffer; } +// LLVMRustModuleNameCallback +pub type ThinLTOModuleNameCallback = + unsafe extern "C" fn(*mut c_void, *const c_char, *const c_char); + /// LLVMRustThinLTOModule #[repr(C)] pub struct ThinLTOModule { @@ -1622,6 +1626,11 @@ extern "C" { Data: &ThinLTOData, Module: &Module, ) -> bool; + pub fn LLVMRustGetThinLTOModuleImports( + Data: *const ThinLTOData, + ModuleNameCallback: ThinLTOModuleNameCallback, + CallbackPayload: *mut c_void, + ); pub fn LLVMRustFreeThinLTOData(Data: &'static mut ThinLTOData); pub fn LLVMRustParseBitcodeForThinLTO( Context: &Context, diff --git a/src/rustllvm/PassWrapper.cpp b/src/rustllvm/PassWrapper.cpp index 09befdaae37c5..5c4bb61781ed1 100644 --- a/src/rustllvm/PassWrapper.cpp +++ b/src/rustllvm/PassWrapper.cpp @@ -1123,6 +1123,28 @@ LLVMRustPrepareThinLTOImport(const LLVMRustThinLTOData *Data, LLVMModuleRef M) { return true; } +extern "C" typedef void (*LLVMRustModuleNameCallback)(void*, // payload + const char*, // importing module name + const char*); // imported module name + +// Calls `module_name_callback` for each module import done by ThinLTO. +// The callback is provided with regular null-terminated C strings. +extern "C" void +LLVMRustGetThinLTOModuleImports(const LLVMRustThinLTOData *data, + LLVMRustModuleNameCallback module_name_callback, + void* callback_payload) { + for (const auto& importing_module : data->ImportLists) { + const std::string importing_module_id = importing_module.getKey().str(); + const auto& imports = importing_module.getValue(); + for (const auto& imported_module : imports) { + const std::string imported_module_id = imported_module.getKey().str(); + module_name_callback(callback_payload, + importing_module_id.c_str(), + imported_module_id.c_str()); + } + } +} + // This struct and various functions are sort of a hack right now, but the // problem is that we've got in-memory LLVM modules after we generate and // optimize all codegen-units for one compilation in rustc. To be compatible @@ -1288,6 +1310,11 @@ LLVMRustPrepareThinLTOImport(const LLVMRustThinLTOData *Data, LLVMModuleRef M) { report_fatal_error("ThinLTO not available"); } +extern "C" LLVMRustThinLTOModuleImports +LLVMRustGetLLVMRustThinLTOModuleImports(const LLVMRustThinLTOData *Data) { + report_fatal_error("ThinLTO not available"); +} + extern "C" void LLVMRustFreeThinLTOData(LLVMRustThinLTOData *Data) { report_fatal_error("ThinLTO not available"); From d97d1e192b37557f760f41cbc9826bb431b14d1a Mon Sep 17 00:00:00 2001 From: Michael Woerister Date: Fri, 17 Aug 2018 16:07:23 +0200 Subject: [PATCH 2/7] Persist ThinLTO import data in incr. comp. session directory. --- src/librustc_codegen_llvm/back/lto.rs | 119 +++++++++++++++++++++++- src/librustc_codegen_llvm/base.rs | 20 ++++ src/librustc_codegen_llvm/lib.rs | 2 +- src/librustc_incremental/lib.rs | 1 + src/librustc_incremental/persist/mod.rs | 1 + 5 files changed, 139 insertions(+), 4 deletions(-) diff --git a/src/librustc_codegen_llvm/back/lto.rs b/src/librustc_codegen_llvm/back/lto.rs index 56858a31efd25..22d862f4ad5d9 100644 --- a/src/librustc_codegen_llvm/back/lto.rs +++ b/src/librustc_codegen_llvm/back/lto.rs @@ -20,16 +20,23 @@ use rustc::hir::def_id::LOCAL_CRATE; use rustc::middle::exported_symbols::SymbolExportLevel; use rustc::session::config::{self, Lto}; use rustc::util::common::time_ext; +use rustc_data_structures::fx::FxHashMap; use time_graph::Timeline; use {ModuleCodegen, ModuleLlvm, ModuleKind, ModuleSource}; use libc; -use std::ffi::CString; +use std::ffi::{CStr, CString}; +use std::fs::File; +use std::io; +use std::mem; +use std::path::Path; use std::ptr; use std::slice; use std::sync::Arc; +pub const THIN_LTO_IMPORTS_INCR_COMP_FILE_NAME: &str = "thin-lto-imports.bin"; + pub fn crate_type_allows_lto(crate_type: config::CrateType) -> bool { match crate_type { config::CrateType::Executable | @@ -199,7 +206,7 @@ pub(crate) fn run(cgcx: &CodegenContext, unreachable!("We should never reach this case if the LTO step \ is deferred to the linker"); } - thin_lto(&diag_handler, modules, upstream_modules, &arr, timeline) + thin_lto(cgcx, &diag_handler, modules, upstream_modules, &arr, timeline) } Lto::No => unreachable!(), } @@ -362,7 +369,8 @@ impl Drop for Linker<'a> { /// calculating the *index* for ThinLTO. This index will then be shared amongst /// all of the `LtoModuleCodegen` units returned below and destroyed once /// they all go out of scope. -fn thin_lto(diag_handler: &Handler, +fn thin_lto(cgcx: &CodegenContext, + diag_handler: &Handler, modules: Vec, serialized_modules: Vec<(SerializedModule, CString)>, symbol_white_list: &[*const libc::c_char], @@ -439,6 +447,17 @@ fn thin_lto(diag_handler: &Handler, write::llvm_err(&diag_handler, "failed to prepare thin LTO context".to_string()) })?; + // Save the ThinLTO import information for incremental compilation. + if let Some(ref incr_comp_session_dir) = cgcx.incr_comp_session_dir { + let path = incr_comp_session_dir.join(THIN_LTO_IMPORTS_INCR_COMP_FILE_NAME); + let imports = ThinLTOImports::from_thin_lto_data(data); + if let Err(err) = imports.save_to_file(&path) { + let msg = format!("Error while writing ThinLTO import data: {}", + err); + return Err(write::llvm_err(&diag_handler, msg)); + } + } + let data = ThinData(data); info!("thin LTO data created"); timeline.record("data"); @@ -776,3 +795,97 @@ impl ThinModule { Ok(module) } } + +#[derive(Debug)] +pub struct ThinLTOImports { + // key = llvm name of importing module, value = list of modules it imports from + imports: FxHashMap>, +} + +impl ThinLTOImports { + pub fn new() -> ThinLTOImports { + ThinLTOImports { + imports: FxHashMap(), + } + } + + /// Load the ThinLTO import map from ThinLTOData. + unsafe fn from_thin_lto_data(data: *const llvm::ThinLTOData) -> ThinLTOImports { + fn module_name_to_str(c_str: &CStr) -> &str { + match c_str.to_str() { + Ok(s) => s, + Err(e) => { + bug!("Encountered non-utf8 LLVM module name `{}`: {}", + c_str.to_string_lossy(), + e) + } + } + } + unsafe extern "C" fn imported_module_callback(payload: *mut libc::c_void, + importing_module_name: *const libc::c_char, + imported_module_name: *const libc::c_char) { + let map = &mut* (payload as *mut ThinLTOImports); + let importing_module_name = CStr::from_ptr(importing_module_name); + let importing_module_name = module_name_to_str(&importing_module_name); + let imported_module_name = CStr::from_ptr(imported_module_name); + let imported_module_name = module_name_to_str(&imported_module_name); + if !map.imports.contains_key(importing_module_name) { + map.imports.insert(importing_module_name.to_owned(), vec![]); + } + map.imports + .get_mut(importing_module_name) + .unwrap() + .push(imported_module_name.to_owned()); + } + let mut map = ThinLTOImports { + imports: FxHashMap(), + }; + llvm::LLVMRustGetThinLTOModuleImports(data, + imported_module_callback, + &mut map as *mut _ as *mut libc::c_void); + map + } + + pub fn save_to_file(&self, path: &Path) -> io::Result<()> { + use std::io::Write; + let file = File::create(path)?; + let mut writer = io::BufWriter::new(file); + for (importing_module_name, imported_modules) in &self.imports { + writeln!(writer, "{}", importing_module_name)?; + for imported_module in imported_modules { + writeln!(writer, " {}", imported_module)?; + } + writeln!(writer)?; + } + Ok(()) + } + + pub fn load_from_file(path: &Path) -> io::Result { + use std::io::BufRead; + let mut imports = FxHashMap(); + let mut current_module = None; + let mut current_imports = vec![]; + let file = File::open(path)?; + for line in io::BufReader::new(file).lines() { + let line = line?; + if line.is_empty() { + let importing_module = current_module + .take() + .expect("Importing module not set"); + imports.insert(importing_module, + mem::replace(&mut current_imports, vec![])); + } else if line.starts_with(" ") { + // This is an imported module + assert_ne!(current_module, None); + current_imports.push(line.trim().to_string()); + } else { + // This is the beginning of a new module + assert_eq!(current_module, None); + current_module = Some(line.trim().to_string()); + } + } + Ok(ThinLTOImports { + imports + }) + } +} \ No newline at end of file diff --git a/src/librustc_codegen_llvm/base.rs b/src/librustc_codegen_llvm/base.rs index 009c6da9d8d15..32f88f8674317 100644 --- a/src/librustc_codegen_llvm/base.rs +++ b/src/librustc_codegen_llvm/base.rs @@ -29,6 +29,7 @@ use super::ModuleCodegen; use super::ModuleKind; use abi; +use back::lto; use back::write::{self, OngoingCodegen}; use llvm::{self, TypeKind, get_param}; use metadata; @@ -1314,6 +1315,25 @@ pub fn visibility_to_llvm(linkage: Visibility) -> llvm::Visibility { } } +#[allow(unused)] +fn load_thin_lto_imports(sess: &Session) -> lto::ThinLTOImports { + let path = rustc_incremental::in_incr_comp_dir_sess( + sess, + lto::THIN_LTO_IMPORTS_INCR_COMP_FILE_NAME + ); + if !path.exists() { + return lto::ThinLTOImports::new(); + } + match lto::ThinLTOImports::load_from_file(&path) { + Ok(imports) => imports, + Err(e) => { + let msg = format!("Error while trying to load ThinLTO import data \ + for incremental compilation: {}", e); + sess.fatal(&msg) + } + } +} + // FIXME(mw): Anything that is produced via DepGraph::with_task() must implement // the HashStable trait. Normally DepGraph::with_task() calls are // hidden behind queries, but CGU creation is a special case in two diff --git a/src/librustc_codegen_llvm/lib.rs b/src/librustc_codegen_llvm/lib.rs index 31eeb5633fba9..e02fd2cae2036 100644 --- a/src/librustc_codegen_llvm/lib.rs +++ b/src/librustc_codegen_llvm/lib.rs @@ -100,7 +100,7 @@ mod back { mod command; pub mod linker; pub mod link; - mod lto; + pub mod lto; pub mod symbol_export; pub mod write; mod rpath; diff --git a/src/librustc_incremental/lib.rs b/src/librustc_incremental/lib.rs index e100b49c7f244..4ffd726c1d47c 100644 --- a/src/librustc_incremental/lib.rs +++ b/src/librustc_incremental/lib.rs @@ -44,6 +44,7 @@ pub use persist::copy_cgu_workproducts_to_incr_comp_cache_dir; pub use persist::save_dep_graph; pub use persist::save_work_product_index; pub use persist::in_incr_comp_dir; +pub use persist::in_incr_comp_dir_sess; pub use persist::prepare_session_directory; pub use persist::finalize_session_directory; pub use persist::delete_workproduct_files; diff --git a/src/librustc_incremental/persist/mod.rs b/src/librustc_incremental/persist/mod.rs index e1f00db56d5cb..17d36ba3fa7f4 100644 --- a/src/librustc_incremental/persist/mod.rs +++ b/src/librustc_incremental/persist/mod.rs @@ -23,6 +23,7 @@ mod file_format; pub use self::fs::finalize_session_directory; pub use self::fs::garbage_collect_session_directories; pub use self::fs::in_incr_comp_dir; +pub use self::fs::in_incr_comp_dir_sess; pub use self::fs::prepare_session_directory; pub use self::load::dep_graph_tcx_init; pub use self::load::load_dep_graph; From 72c1993b8e9ec095bab299b4cc298be7eb9bf1ee Mon Sep 17 00:00:00 2001 From: Michael Woerister Date: Mon, 20 Aug 2018 13:42:07 +0200 Subject: [PATCH 3/7] Make codegen not be a query (since it's not a real query anyway). --- src/librustc/ty/query/config.rs | 6 ---- src/librustc/ty/query/mod.rs | 3 +- src/librustc_codegen_llvm/base.rs | 35 ++++++++++++------- src/librustc_mir/monomorphize/partitioning.rs | 6 +++- 4 files changed, 28 insertions(+), 22 deletions(-) diff --git a/src/librustc/ty/query/config.rs b/src/librustc/ty/query/config.rs index b5093d0a1fc95..07b1863e32c6c 100644 --- a/src/librustc/ty/query/config.rs +++ b/src/librustc/ty/query/config.rs @@ -722,12 +722,6 @@ impl<'tcx> QueryDescription<'tcx> for queries::codegen_unit<'tcx> { } } -impl<'tcx> QueryDescription<'tcx> for queries::compile_codegen_unit<'tcx> { - fn describe(_tcx: TyCtxt, _: InternedString) -> String { - "compile_codegen_unit".to_string() - } -} - impl<'tcx> QueryDescription<'tcx> for queries::output_filenames<'tcx> { fn describe(_tcx: TyCtxt, _: CrateNum) -> String { "output_filenames".to_string() diff --git a/src/librustc/ty/query/mod.rs b/src/librustc/ty/query/mod.rs index 6f61583e49b8e..9485f62c61ca0 100644 --- a/src/librustc/ty/query/mod.rs +++ b/src/librustc/ty/query/mod.rs @@ -28,7 +28,7 @@ use middle::lib_features::LibFeatures; use middle::lang_items::{LanguageItems, LangItem}; use middle::exported_symbols::{SymbolExportLevel, ExportedSymbol}; use mir::interpret::ConstEvalResult; -use mir::mono::{CodegenUnit, Stats}; +use mir::mono::CodegenUnit; use mir; use mir::interpret::{GlobalId, Allocation}; use session::{CompileResult, CrateDisambiguator}; @@ -530,7 +530,6 @@ define_queries! { <'tcx> -> (Arc, Arc>>>), [] fn is_codegened_item: IsCodegenedItem(DefId) -> bool, [] fn codegen_unit: CodegenUnit(InternedString) -> Arc>, - [] fn compile_codegen_unit: CompileCodegenUnit(InternedString) -> Stats, }, Other { diff --git a/src/librustc_codegen_llvm/base.rs b/src/librustc_codegen_llvm/base.rs index 32f88f8674317..59058579f643e 100644 --- a/src/librustc_codegen_llvm/base.rs +++ b/src/librustc_codegen_llvm/base.rs @@ -891,7 +891,7 @@ pub fn codegen_crate<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, &format!("codegen {}", cgu.name())) }); let start_time = Instant::now(); - all_stats.extend(tcx.compile_codegen_unit(*cgu.name())); + all_stats.extend(compile_codegen_unit(tcx, *cgu.name())); total_codegen_time += start_time.elapsed(); ongoing_codegen.check_for_errors(tcx.sess); } @@ -1157,11 +1157,15 @@ fn is_codegened_item(tcx: TyCtxt, id: DefId) -> bool { } fn compile_codegen_unit<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, - cgu: InternedString) -> Stats { - let cgu = tcx.codegen_unit(cgu); - + cgu_name: InternedString) + -> Stats { let start_time = Instant::now(); - let (stats, module) = module_codegen(tcx, cgu); + + let dep_node = tcx.codegen_unit(cgu_name).codegen_dep_node(tcx); + let ((stats, module), _) = tcx.dep_graph.with_task(dep_node, + tcx, + cgu_name, + module_codegen); let time_to_codegen = start_time.elapsed(); // We assume that the cost to run LLVM on a CGU is proportional to @@ -1170,23 +1174,29 @@ fn compile_codegen_unit<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, time_to_codegen.subsec_nanos() as u64; write::submit_codegened_module_to_llvm(tcx, - module, - cost); + module, + cost); + + if tcx.dep_graph.is_fully_enabled() { + let dep_node_index = tcx.dep_graph.dep_node_index_of(&dep_node); + tcx.dep_graph.mark_loaded_from_cache(dep_node_index, false); + } + return stats; fn module_codegen<'a, 'tcx>( tcx: TyCtxt<'a, 'tcx, 'tcx>, - cgu: Arc>) + cgu_name: InternedString) -> (Stats, ModuleCodegen) { - let cgu_name = cgu.name().to_string(); + let cgu = tcx.codegen_unit(cgu_name); // Instantiate monomorphizations without filling out definitions yet... - let llvm_module = ModuleLlvm::new(tcx.sess, &cgu_name); + let llvm_module = ModuleLlvm::new(tcx.sess, &cgu_name.as_str()); let stats = { let cx = CodegenCx::new(tcx, cgu, &llvm_module); let mono_items = cx.codegen_unit - .items_in_deterministic_order(cx.tcx); + .items_in_deterministic_order(cx.tcx); for &(mono_item, (linkage, visibility)) in &mono_items { mono_item.predefine(&cx, linkage, visibility); } @@ -1235,7 +1245,7 @@ fn compile_codegen_unit<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, }; (stats, ModuleCodegen { - name: cgu_name, + name: cgu_name.to_string(), source: ModuleSource::Codegened(llvm_module), kind: ModuleKind::Regular, }) @@ -1255,7 +1265,6 @@ pub fn provide(providers: &mut Providers) { .cloned() .unwrap_or_else(|| panic!("failed to find cgu with name {:?}", name)) }; - providers.compile_codegen_unit = compile_codegen_unit; provide_extern(providers); } diff --git a/src/librustc_mir/monomorphize/partitioning.rs b/src/librustc_mir/monomorphize/partitioning.rs index c480fa4124665..a8c27c037513f 100644 --- a/src/librustc_mir/monomorphize/partitioning.rs +++ b/src/librustc_mir/monomorphize/partitioning.rs @@ -103,7 +103,7 @@ //! inlining, even when they are not marked #[inline]. use monomorphize::collector::InliningMap; -use rustc::dep_graph::WorkProductId; +use rustc::dep_graph::{WorkProductId, DepNode, DepConstructor}; use rustc::hir::CodegenFnAttrFlags; use rustc::hir::def_id::{DefId, LOCAL_CRATE, CRATE_DEF_INDEX}; use rustc::hir::map::DefPathData; @@ -194,6 +194,10 @@ pub trait CodegenUnitExt<'tcx> { items.sort_by_cached_key(|&(i, _)| item_sort_key(tcx, i)); items } + + fn codegen_dep_node(&self, tcx: TyCtxt<'_, 'tcx, 'tcx>) -> DepNode { + DepNode::new(tcx, DepConstructor::CompileCodegenUnit(self.name().clone())) + } } impl<'tcx> CodegenUnitExt<'tcx> for CodegenUnit<'tcx> { From 64a738d8ce457b8d9b3a750ca61835214b6b438c Mon Sep 17 00:00:00 2001 From: Michael Woerister Date: Mon, 20 Aug 2018 17:13:01 +0200 Subject: [PATCH 4/7] Support local ThinLTO with incremental compilation. --- src/librustc/dep_graph/graph.rs | 3 +- src/librustc/session/config.rs | 2 +- src/librustc/session/mod.rs | 19 +- src/librustc_codegen_llvm/Cargo.toml | 1 + src/librustc_codegen_llvm/back/lto.rs | 108 +++- src/librustc_codegen_llvm/back/write.rs | 541 +++++++++++++----- src/librustc_codegen_llvm/base.rs | 174 ++++-- src/librustc_codegen_llvm/lib.rs | 40 +- .../persist/work_product.rs | 1 + src/librustc_mir/monomorphize/partitioning.rs | 11 +- 10 files changed, 634 insertions(+), 266 deletions(-) diff --git a/src/librustc/dep_graph/graph.rs b/src/librustc/dep_graph/graph.rs index a285399657177..1c883eeed966c 100644 --- a/src/librustc/dep_graph/graph.rs +++ b/src/librustc/dep_graph/graph.rs @@ -878,11 +878,12 @@ pub struct WorkProduct { pub saved_files: Vec<(WorkProductFileKind, String)>, } -#[derive(Clone, Copy, Debug, RustcEncodable, RustcDecodable)] +#[derive(Clone, Copy, Debug, RustcEncodable, RustcDecodable, PartialEq)] pub enum WorkProductFileKind { Object, Bytecode, BytecodeCompressed, + PreThinLtoBytecode, } pub(super) struct CurrentDepGraph { diff --git a/src/librustc/session/config.rs b/src/librustc/session/config.rs index ee683e37648f0..ee3fabc58d53f 100644 --- a/src/librustc/session/config.rs +++ b/src/librustc/session/config.rs @@ -68,7 +68,7 @@ pub enum OptLevel { SizeMin, // -Oz } -#[derive(Clone, Copy, PartialEq, Hash)] +#[derive(Clone, Copy, PartialEq, Hash, Debug)] pub enum Lto { /// Don't do any LTO whatsoever No, diff --git a/src/librustc/session/mod.rs b/src/librustc/session/mod.rs index 00c5369e0644e..778c388c7dec7 100644 --- a/src/librustc/session/mod.rs +++ b/src/librustc/session/mod.rs @@ -580,11 +580,6 @@ impl Session { return config::Lto::No; } - // Right now ThinLTO isn't compatible with incremental compilation. - if self.opts.incremental.is_some() { - return config::Lto::No; - } - // Now we're in "defaults" territory. By default we enable ThinLTO for // optimized compiles (anything greater than O0). match self.opts.optimize { @@ -1177,8 +1172,18 @@ pub fn build_session_( // commandline argument, you can do so here. fn validate_commandline_args_with_session_available(sess: &Session) { - if sess.lto() != Lto::No && sess.opts.incremental.is_some() { - sess.err("can't perform LTO when compiling incrementally"); + if sess.opts.incremental.is_some() { + match sess.lto() { + Lto::Yes | + Lto::Thin | + Lto::Fat => { + sess.err("can't perform LTO when compiling incrementally"); + } + Lto::ThinLocal | + Lto::No => { + // This is fine + } + } } // Since we don't know if code in an rlib will be linked to statically or diff --git a/src/librustc_codegen_llvm/Cargo.toml b/src/librustc_codegen_llvm/Cargo.toml index 28fa49846b736..b711502b14b7f 100644 --- a/src/librustc_codegen_llvm/Cargo.toml +++ b/src/librustc_codegen_llvm/Cargo.toml @@ -14,6 +14,7 @@ cc = "1.0.1" num_cpus = "1.0" rustc-demangle = "0.1.4" rustc_llvm = { path = "../librustc_llvm" } +memmap = "0.6" [features] # This is used to convince Cargo to separately cache builds of `rustc_codegen_llvm` diff --git a/src/librustc_codegen_llvm/back/lto.rs b/src/librustc_codegen_llvm/back/lto.rs index 22d862f4ad5d9..9813134992760 100644 --- a/src/librustc_codegen_llvm/back/lto.rs +++ b/src/librustc_codegen_llvm/back/lto.rs @@ -16,13 +16,14 @@ use errors::{FatalError, Handler}; use llvm::archive_ro::ArchiveRO; use llvm::{True, False}; use llvm; +use memmap; use rustc::hir::def_id::LOCAL_CRATE; use rustc::middle::exported_symbols::SymbolExportLevel; use rustc::session::config::{self, Lto}; use rustc::util::common::time_ext; -use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use time_graph::Timeline; -use {ModuleCodegen, ModuleLlvm, ModuleKind, ModuleSource}; +use {ModuleCodegen, ModuleLlvm, ModuleKind}; use libc; @@ -82,8 +83,8 @@ impl LtoModuleCodegen { let module = module.take().unwrap(); { let config = cgcx.config(module.kind); - let llmod = module.llvm().unwrap().llmod(); - let tm = &*module.llvm().unwrap().tm; + let llmod = module.module_llvm.llmod(); + let tm = &*module.module_llvm.tm; run_pass_manager(cgcx, tm, llmod, config, false); timeline.record("fat-done"); } @@ -106,6 +107,7 @@ impl LtoModuleCodegen { pub(crate) fn run(cgcx: &CodegenContext, modules: Vec, + import_only_modules: Vec<(SerializedModule, CString)>, timeline: &mut Timeline) -> Result, FatalError> { @@ -194,11 +196,19 @@ pub(crate) fn run(cgcx: &CodegenContext, } } - let arr = symbol_white_list.iter().map(|c| c.as_ptr()).collect::>(); + let symbol_white_list = symbol_white_list.iter() + .map(|c| c.as_ptr()) + .collect::>(); match cgcx.lto { Lto::Yes | // `-C lto` == fat LTO by default Lto::Fat => { - fat_lto(cgcx, &diag_handler, modules, upstream_modules, &arr, timeline) + assert!(import_only_modules.is_empty()); + fat_lto(cgcx, + &diag_handler, + modules, + upstream_modules, + &symbol_white_list, + timeline) } Lto::Thin | Lto::ThinLocal => { @@ -206,7 +216,13 @@ pub(crate) fn run(cgcx: &CodegenContext, unreachable!("We should never reach this case if the LTO step \ is deferred to the linker"); } - thin_lto(cgcx, &diag_handler, modules, upstream_modules, &arr, timeline) + thin_lto(cgcx, + &diag_handler, + modules, + upstream_modules, + import_only_modules, + &symbol_white_list, + timeline) } Lto::No => unreachable!(), } @@ -236,7 +252,7 @@ fn fat_lto(cgcx: &CodegenContext, .filter(|&(_, module)| module.kind == ModuleKind::Regular) .map(|(i, module)| { let cost = unsafe { - llvm::LLVMRustModuleCost(module.llvm().unwrap().llmod()) + llvm::LLVMRustModuleCost(module.module_llvm.llmod()) }; (cost, i) }) @@ -246,7 +262,7 @@ fn fat_lto(cgcx: &CodegenContext, let mut serialized_bitcode = Vec::new(); { let (llcx, llmod) = { - let llvm = module.llvm().expect("can't lto pre-codegened modules"); + let llvm = &module.module_llvm; (&llvm.llcx, llvm.llmod()) }; info!("using {:?} as a base module", module.name); @@ -262,8 +278,7 @@ fn fat_lto(cgcx: &CodegenContext, // way we know of to do that is to serialize them to a string and them parse // them later. Not great but hey, that's why it's "fat" LTO, right? for module in modules { - let llvm = module.llvm().expect("can't lto pre-codegened modules"); - let buffer = ModuleBuffer::new(llvm.llmod()); + let buffer = ModuleBuffer::new(module.module_llvm.llmod()); let llmod_id = CString::new(&module.name[..]).unwrap(); serialized_modules.push((SerializedModule::Local(buffer), llmod_id)); } @@ -373,6 +388,7 @@ fn thin_lto(cgcx: &CodegenContext, diag_handler: &Handler, modules: Vec, serialized_modules: Vec<(SerializedModule, CString)>, + import_only_modules: Vec<(SerializedModule, CString)>, symbol_white_list: &[*const libc::c_char], timeline: &mut Timeline) -> Result, FatalError> @@ -393,9 +409,8 @@ fn thin_lto(cgcx: &CodegenContext, // analysis! for (i, module) in modules.iter().enumerate() { info!("local module: {} - {}", i, module.name); - let llvm = module.llvm().expect("can't lto precodegened module"); let name = CString::new(module.name.clone()).unwrap(); - let buffer = ThinBuffer::new(llvm.llmod()); + let buffer = ThinBuffer::new(module.module_llvm.llmod()); thin_modules.push(llvm::ThinLTOModule { identifier: name.as_ptr(), data: buffer.data().as_ptr(), @@ -434,6 +449,22 @@ fn thin_lto(cgcx: &CodegenContext, module_names.push(name); } + // All the modules collected up to this point we actually want to + // optimize. The `import_only_modules` below need to be in the list of + // available modules but we don't need to run optimizations for them + // since we already have their optimized version cached. + let modules_to_optimize = module_names.len(); + for (module, name) in import_only_modules { + info!("foreign module {:?}", name); + thin_modules.push(llvm::ThinLTOModule { + identifier: name.as_ptr(), + data: module.data().as_ptr(), + len: module.data().len(), + }); + serialized.push(module); + module_names.push(name); + } + // Delegate to the C++ bindings to create some data here. Once this is a // tried-and-true interface we may wish to try to upstream some of this // to LLVM itself, right now we reimplement a lot of what they do @@ -450,7 +481,21 @@ fn thin_lto(cgcx: &CodegenContext, // Save the ThinLTO import information for incremental compilation. if let Some(ref incr_comp_session_dir) = cgcx.incr_comp_session_dir { let path = incr_comp_session_dir.join(THIN_LTO_IMPORTS_INCR_COMP_FILE_NAME); - let imports = ThinLTOImports::from_thin_lto_data(data); + + // The import information from the current compilation session. It + // does not contain info about modules that have been loaded from + // the cache instead of having been recompiled... + let current_imports = ThinLTOImports::from_thin_lto_data(data); + + // ... so we load this additional information from the previous + // cache file if necessary. + let imports = if path.exists() { + let prev_imports = ThinLTOImports::load_from_file(&path).unwrap(); + prev_imports.update(current_imports, &module_names) + } else { + current_imports + }; + if let Err(err) = imports.save_to_file(&path) { let msg = format!("Error while writing ThinLTO import data: {}", err); @@ -472,7 +517,7 @@ fn thin_lto(cgcx: &CodegenContext, serialized_modules: serialized, module_names, }); - Ok((0..shared.module_names.len()).map(|i| { + Ok((0..modules_to_optimize).map(|i| { LtoModuleCodegen::Thin(ThinModule { shared: shared.clone(), idx: i, @@ -546,6 +591,7 @@ fn run_pass_manager(cgcx: &CodegenContext, pub enum SerializedModule { Local(ModuleBuffer), FromRlib(Vec), + FromUncompressedFile(memmap::Mmap, File), } impl SerializedModule { @@ -553,6 +599,7 @@ impl SerializedModule { match *self { SerializedModule::Local(ref m) => m.data(), SerializedModule::FromRlib(ref m) => m, + SerializedModule::FromUncompressedFile(ref m, _) => m, } } } @@ -682,16 +729,16 @@ impl ThinModule { write::llvm_err(&diag_handler, msg) })? as *const _; let module = ModuleCodegen { - source: ModuleSource::Codegened(ModuleLlvm { + module_llvm: ModuleLlvm { llmod_raw, llcx, tm, - }), + }, name: self.name().to_string(), kind: ModuleKind::Regular, }; { - let llmod = module.llvm().unwrap().llmod(); + let llmod = module.module_llvm.llmod(); cgcx.save_temp_bitcode(&module, "thin-lto-input"); // Before we do much else find the "main" `DICompileUnit` that we'll be @@ -787,7 +834,7 @@ impl ThinModule { // little differently. info!("running thin lto passes over {}", module.name); let config = cgcx.config(module.kind); - run_pass_manager(cgcx, module.llvm().unwrap().tm, llmod, config, true); + run_pass_manager(cgcx, module.module_llvm.tm, llmod, config, true); cgcx.save_temp_bitcode(&module, "thin-lto-after-pm"); timeline.record("thin-done"); } @@ -809,6 +856,26 @@ impl ThinLTOImports { } } + pub fn modules_imported_by(&self, llvm_module_name: &str) -> &[String] { + self.imports.get(llvm_module_name).map(|v| &v[..]).unwrap_or(&[]) + } + + pub fn update(mut self, new: ThinLTOImports, module_names: &[CString]) -> ThinLTOImports { + let module_names: FxHashSet<_> = module_names.iter().map(|name| { + name.clone().into_string().unwrap() + }).collect(); + + // Remove all modules that don't exist anymore. + self.imports.retain(|k, _| module_names.contains(k)); + + // Overwrite old values + for (importing_module, imported_modules) in new.imports { + self.imports.insert(importing_module, imported_modules); + } + + self + } + /// Load the ThinLTO import map from ThinLTOData. unsafe fn from_thin_lto_data(data: *const llvm::ThinLTOData) -> ThinLTOImports { fn module_name_to_str(c_str: &CStr) -> &str { @@ -832,6 +899,7 @@ impl ThinLTOImports { if !map.imports.contains_key(importing_module_name) { map.imports.insert(importing_module_name.to_owned(), vec![]); } + map.imports .get_mut(importing_module_name) .unwrap() @@ -888,4 +956,4 @@ impl ThinLTOImports { imports }) } -} \ No newline at end of file +} diff --git a/src/librustc_codegen_llvm/back/write.rs b/src/librustc_codegen_llvm/back/write.rs index 2373428d68c0d..e1d69db83b99d 100644 --- a/src/librustc_codegen_llvm/back/write.rs +++ b/src/librustc_codegen_llvm/back/write.rs @@ -10,14 +10,16 @@ use attributes; use back::bytecode::{self, RLIB_BYTECODE_EXTENSION}; -use back::lto::{self, ModuleBuffer, ThinBuffer}; +use back::lto::{self, ModuleBuffer, ThinBuffer, SerializedModule}; use back::link::{self, get_linker, remove}; use back::command::Command; use back::linker::LinkerInfo; use back::symbol_export::ExportedSymbols; use base; use consts; -use rustc_incremental::{copy_cgu_workproducts_to_incr_comp_cache_dir, in_incr_comp_dir}; +use memmap; +use rustc_incremental::{copy_cgu_workproducts_to_incr_comp_cache_dir, + in_incr_comp_dir, in_incr_comp_dir_sess}; use rustc::dep_graph::{WorkProduct, WorkProductId, WorkProductFileKind}; use rustc::middle::cstore::EncodedMetadata; use rustc::session::config::{self, OutputFilenames, OutputType, Passes, Sanitizer, Lto}; @@ -26,7 +28,8 @@ use rustc::util::nodemap::FxHashMap; use time_graph::{self, TimeGraph, Timeline}; use llvm::{self, DiagnosticInfo, PassManager, SMDiagnostic}; use llvm_util; -use {CodegenResults, ModuleSource, ModuleCodegen, CompiledModule, ModuleKind}; +use {CodegenResults, ModuleCodegen, CompiledModule, ModuleKind, ModuleLlvm, + CachedModuleCodegen}; use CrateInfo; use rustc::hir::def_id::{CrateNum, LOCAL_CRATE}; use rustc::ty::TyCtxt; @@ -84,6 +87,8 @@ pub const TLS_MODEL_ARGS : [(&'static str, llvm::ThreadLocalMode); 4] = [ ("local-exec", llvm::ThreadLocalMode::LocalExec), ]; +const PRE_THIN_LTO_BC_EXT: &str = "pre-thin-lto.bc"; + pub fn llvm_err(handler: &errors::Handler, msg: String) -> FatalError { match llvm::last_error() { Some(err) => handler.fatal(&format!("{}: {}", msg, err)), @@ -224,6 +229,7 @@ pub struct ModuleConfig { // Flags indicating which outputs to produce. emit_no_opt_bc: bool, + emit_pre_thin_lto_bc: bool, emit_bc: bool, emit_bc_compressed: bool, emit_lto_bc: bool, @@ -260,6 +266,7 @@ impl ModuleConfig { pgo_use: String::new(), emit_no_opt_bc: false, + emit_pre_thin_lto_bc: false, emit_bc: false, emit_bc_compressed: false, emit_lto_bc: false, @@ -392,7 +399,7 @@ impl CodegenContext { let cgu = Some(&module.name[..]); let path = self.output_filenames.temp_path_ext(&ext, cgu); let cstr = path2cstr(&path); - let llmod = module.llvm().unwrap().llmod(); + let llmod = module.module_llvm.llmod(); llvm::LLVMWriteBitcodeToFile(llmod, cstr.as_ptr()); } } @@ -495,13 +502,9 @@ unsafe fn optimize(cgcx: &CodegenContext, timeline: &mut Timeline) -> Result<(), FatalError> { - let (llmod, llcx, tm) = match module.source { - ModuleSource::Codegened(ref llvm) => (llvm.llmod(), &*llvm.llcx, &*llvm.tm), - ModuleSource::Preexisting(_) => { - bug!("optimize_and_codegen: called with ModuleSource::Preexisting") - } - }; - + let llmod = module.module_llvm.llmod(); + let llcx = &*module.module_llvm.llcx; + let tm = &*module.module_llvm.tm; let _handlers = DiagnosticHandlers::new(cgcx, diag_handler, llcx); let module_name = module.name.clone(); @@ -622,12 +625,20 @@ unsafe fn optimize(cgcx: &CodegenContext, // Deallocate managers that we're now done with llvm::LLVMDisposePassManager(fpm); llvm::LLVMDisposePassManager(mpm); + + if config.emit_pre_thin_lto_bc { + let out = cgcx.output_filenames.temp_path_ext(PRE_THIN_LTO_BC_EXT, + module_name); + let out = path2cstr(&out); + llvm::LLVMWriteBitcodeToFile(llmod, out.as_ptr()); + } } Ok(()) } fn generate_lto_work(cgcx: &CodegenContext, - modules: Vec) + modules: Vec, + import_only_modules: Vec<(SerializedModule, CString)>) -> Vec<(WorkItem, u64)> { let mut timeline = cgcx.time_graph.as_ref().map(|tg| { @@ -635,7 +646,7 @@ fn generate_lto_work(cgcx: &CodegenContext, CODEGEN_WORK_PACKAGE_KIND, "generate lto") }).unwrap_or(Timeline::noop()); - let lto_modules = lto::run(cgcx, modules, &mut timeline) + let lto_modules = lto::run(cgcx, modules, import_only_modules, &mut timeline) .unwrap_or_else(|e| e.raise()); lto_modules.into_iter().map(|module| { @@ -653,12 +664,9 @@ unsafe fn codegen(cgcx: &CodegenContext, { timeline.record("codegen"); { - let (llmod, llcx, tm) = match module.source { - ModuleSource::Codegened(ref llvm) => (llvm.llmod(), &*llvm.llcx, &*llvm.tm), - ModuleSource::Preexisting(_) => { - bug!("codegen: called with ModuleSource::Preexisting") - } - }; + let llmod = module.module_llvm.llmod(); + let llcx = &*module.module_llvm.llcx; + let tm = &*module.module_llvm.tm; let module_name = module.name.clone(); let module_name = Some(&module_name[..]); let handlers = DiagnosticHandlers::new(cgcx, diag_handler, llcx); @@ -912,6 +920,20 @@ fn need_crate_bitcode_for_rlib(sess: &Session) -> bool { sess.opts.output_types.contains_key(&OutputType::Exe) } +fn need_pre_thin_lto_bitcode_for_incr_comp(sess: &Session) -> bool { + if sess.opts.incremental.is_none() { + return false + } + + match sess.lto() { + Lto::Yes | + Lto::Fat | + Lto::No => false, + Lto::Thin | + Lto::ThinLocal => true, + } +} + pub fn start_async_codegen(tcx: TyCtxt, time_graph: Option, metadata: EncodedMetadata, @@ -970,6 +992,7 @@ pub fn start_async_codegen(tcx: TyCtxt, // Save all versions of the bytecode if we're saving our temporaries. if sess.opts.cg.save_temps { modules_config.emit_no_opt_bc = true; + modules_config.emit_pre_thin_lto_bc = true; modules_config.emit_bc = true; modules_config.emit_lto_bc = true; metadata_config.emit_bc = true; @@ -984,6 +1007,9 @@ pub fn start_async_codegen(tcx: TyCtxt, allocator_config.emit_bc_compressed = true; } + modules_config.emit_pre_thin_lto_bc = + need_pre_thin_lto_bitcode_for_incr_comp(sess); + modules_config.no_integrated_as = tcx.sess.opts.cg.no_integrated_as || tcx.sess.target.target.options.no_integrated_as; @@ -1056,7 +1082,8 @@ pub fn start_async_codegen(tcx: TyCtxt, fn copy_all_cgu_workproducts_to_incr_comp_cache_dir( sess: &Session, - compiled_modules: &CompiledModules + compiled_modules: &CompiledModules, + output_filenames: &OutputFilenames, ) -> FxHashMap { let mut work_products = FxHashMap::default(); @@ -1064,7 +1091,7 @@ fn copy_all_cgu_workproducts_to_incr_comp_cache_dir( return work_products; } - for module in compiled_modules.modules.iter() { + for module in compiled_modules.modules.iter().filter(|m| m.kind == ModuleKind::Regular) { let mut files = vec![]; if let Some(ref path) = module.object { @@ -1077,6 +1104,13 @@ fn copy_all_cgu_workproducts_to_incr_comp_cache_dir( files.push((WorkProductFileKind::BytecodeCompressed, path.clone())); } + let pre_thin_lto_bytecode_path = + output_filenames.temp_path_ext(PRE_THIN_LTO_BC_EXT, Some(&module.name)); + + if pre_thin_lto_bytecode_path.exists() { + files.push((WorkProductFileKind::PreThinLtoBytecode, pre_thin_lto_bytecode_path)); + } + if let Some((id, product)) = copy_cgu_workproducts_to_incr_comp_cache_dir(sess, &module.name, &files) { work_products.insert(id, product); @@ -1236,21 +1270,34 @@ fn produce_final_output_artifacts(sess: &Session, // These are used in linking steps and will be cleaned up afterward. } -pub(crate) fn dump_incremental_data(codegen_results: &CodegenResults) { - println!("[incremental] Re-using {} out of {} modules", - codegen_results.modules.iter().filter(|m| m.pre_existing).count(), - codegen_results.modules.len()); +pub(crate) fn dump_incremental_data(_codegen_results: &CodegenResults) { + // FIXME(mw): This does not work at the moment because the situation has + // become more complicated due to incremental LTO. Now a CGU + // can have more than two caching states. + // println!("[incremental] Re-using {} out of {} modules", + // codegen_results.modules.iter().filter(|m| m.pre_existing).count(), + // codegen_results.modules.len()); } enum WorkItem { + /// Optimize a newly codegened, totally unoptimized module. Optimize(ModuleCodegen), + /// Copy the post-LTO artifacts from the incremental cache to the output + /// directory. + CopyPostLtoArtifacts(CachedModuleCodegen), + /// Load the pre-LTO version of a module from the incremental cache, so it + /// can be run through LTO again. + LoadPreLtoModule(CachedModuleCodegen), + /// Perform (Thin)LTO on the given module. LTO(lto::LtoModuleCodegen), } impl WorkItem { - fn kind(&self) -> ModuleKind { + fn module_kind(&self) -> ModuleKind { match *self { WorkItem::Optimize(ref m) => m.kind, + WorkItem::CopyPostLtoArtifacts(_) | + WorkItem::LoadPreLtoModule(_) | WorkItem::LTO(_) => ModuleKind::Regular, } } @@ -1258,6 +1305,8 @@ impl WorkItem { fn name(&self) -> String { match *self { WorkItem::Optimize(ref m) => format!("optimize: {}", m.name), + WorkItem::LoadPreLtoModule(ref m) => format!("load pre-lto module: {}", m.name), + WorkItem::CopyPostLtoArtifacts(ref m) => format!("copy post LTO artifacts: {}", m.name), WorkItem::LTO(ref m) => format!("lto: {}", m.name()), } } @@ -1273,141 +1322,254 @@ fn execute_work_item(cgcx: &CodegenContext, timeline: &mut Timeline) -> Result { + match work_item { + work_item @ WorkItem::Optimize(_) => { + execute_optimize_work_item(cgcx, work_item, timeline) + } + work_item @ WorkItem::LoadPreLtoModule(_) => { + execute_load_pre_lto_mod_work_item(cgcx, work_item, timeline) + } + work_item @ WorkItem::CopyPostLtoArtifacts(_) => { + execute_copy_from_cache_work_item(cgcx, work_item, timeline) + } + work_item @ WorkItem::LTO(_) => { + execute_lto_work_item(cgcx, work_item, timeline) + } + } +} + +fn execute_optimize_work_item(cgcx: &CodegenContext, + work_item: WorkItem, + timeline: &mut Timeline) + -> Result +{ + let config = cgcx.config(work_item.module_kind()); + + let module = if let WorkItem::Optimize(module) = work_item { + module + } else { + bug!("execute_optimize_work_item() called with non-WorkItem::Optimize"); + }; + let diag_handler = cgcx.create_diag_handler(); - let config = cgcx.config(work_item.kind()); - let module = match work_item { - WorkItem::Optimize(module) => module, - WorkItem::LTO(mut lto) => { - unsafe { - let module = lto.optimize(cgcx, timeline)?; - let module = codegen(cgcx, &diag_handler, module, config, timeline)?; - return Ok(WorkItemResult::Compiled(module)) - } + + unsafe { + optimize(cgcx, &diag_handler, &module, config, timeline)?; + } + + let linker_does_lto = cgcx.opts.debugging_opts.cross_lang_lto.enabled(); + + // After we've done the initial round of optimizations we need to + // decide whether to synchronously codegen this module or ship it + // back to the coordinator thread for further LTO processing (which + // has to wait for all the initial modules to be optimized). + // + // Here we dispatch based on the `cgcx.lto` and kind of module we're + // codegenning... + let needs_lto = match cgcx.lto { + Lto::No => false, + + // If the linker does LTO, we don't have to do it. Note that we + // keep doing full LTO, if it is requested, as not to break the + // assumption that the output will be a single module. + Lto::Thin | Lto::ThinLocal if linker_does_lto => false, + + // Here we've got a full crate graph LTO requested. We ignore + // this, however, if the crate type is only an rlib as there's + // no full crate graph to process, that'll happen later. + // + // This use case currently comes up primarily for targets that + // require LTO so the request for LTO is always unconditionally + // passed down to the backend, but we don't actually want to do + // anything about it yet until we've got a final product. + Lto::Yes | Lto::Fat | Lto::Thin => { + cgcx.crate_types.len() != 1 || + cgcx.crate_types[0] != config::CrateType::Rlib + } + + // When we're automatically doing ThinLTO for multi-codegen-unit + // builds we don't actually want to LTO the allocator modules if + // it shows up. This is due to various linker shenanigans that + // we'll encounter later. + // + // Additionally here's where we also factor in the current LLVM + // version. If it doesn't support ThinLTO we skip this. + Lto::ThinLocal => { + module.kind != ModuleKind::Allocator && + unsafe { llvm::LLVMRustThinLTOAvailable() } } }; - let module_name = module.name.clone(); - let pre_existing = match module.source { - ModuleSource::Codegened(_) => None, - ModuleSource::Preexisting(ref wp) => Some(wp.clone()), + // Metadata modules never participate in LTO regardless of the lto + // settings. + let needs_lto = needs_lto && module.kind != ModuleKind::Metadata; + + if needs_lto { + Ok(WorkItemResult::NeedsLTO(module)) + } else { + let module = unsafe { + codegen(cgcx, &diag_handler, module, config, timeline)? + }; + Ok(WorkItemResult::Compiled(module)) + } +} + +fn execute_copy_from_cache_work_item(cgcx: &CodegenContext, + work_item: WorkItem, + _: &mut Timeline) + -> Result +{ + let config = cgcx.config(work_item.module_kind()); + + let module = if let WorkItem::CopyPostLtoArtifacts(module) = work_item { + module + } else { + bug!("execute_copy_from_cache_work_item() called with wrong WorkItem kind.") }; - if let Some(wp) = pre_existing { - let incr_comp_session_dir = cgcx.incr_comp_session_dir - .as_ref() - .unwrap(); - let name = &module.name; - let mut object = None; - let mut bytecode = None; - let mut bytecode_compressed = None; - for (kind, saved_file) in wp.saved_files { - let obj_out = match kind { - WorkProductFileKind::Object => { - let path = cgcx.output_filenames.temp_path(OutputType::Object, Some(name)); - object = Some(path.clone()); - path - } - WorkProductFileKind::Bytecode => { - let path = cgcx.output_filenames.temp_path(OutputType::Bitcode, Some(name)); - bytecode = Some(path.clone()); - path - } - WorkProductFileKind::BytecodeCompressed => { - let path = cgcx.output_filenames.temp_path(OutputType::Bitcode, Some(name)) - .with_extension(RLIB_BYTECODE_EXTENSION); - bytecode_compressed = Some(path.clone()); - path - } - }; - let source_file = in_incr_comp_dir(&incr_comp_session_dir, - &saved_file); - debug!("copying pre-existing module `{}` from {:?} to {}", - module.name, - source_file, - obj_out.display()); - match link_or_copy(&source_file, &obj_out) { - Ok(_) => { } - Err(err) => { - diag_handler.err(&format!("unable to copy {} to {}: {}", - source_file.display(), - obj_out.display(), - err)); - } + let incr_comp_session_dir = cgcx.incr_comp_session_dir + .as_ref() + .unwrap(); + let mut object = None; + let mut bytecode = None; + let mut bytecode_compressed = None; + for (kind, saved_file) in &module.source.saved_files { + let obj_out = match kind { + WorkProductFileKind::Object => { + let path = cgcx.output_filenames.temp_path(OutputType::Object, + Some(&module.name)); + object = Some(path.clone()); + path + } + WorkProductFileKind::Bytecode => { + let path = cgcx.output_filenames.temp_path(OutputType::Bitcode, + Some(&module.name)); + bytecode = Some(path.clone()); + path + } + WorkProductFileKind::BytecodeCompressed => { + let path = cgcx.output_filenames.temp_path(OutputType::Bitcode, + Some(&module.name)) + .with_extension(RLIB_BYTECODE_EXTENSION); + bytecode_compressed = Some(path.clone()); + path + } + WorkProductFileKind::PreThinLtoBytecode => { + continue; + } + }; + let source_file = in_incr_comp_dir(&incr_comp_session_dir, + &saved_file); + debug!("copying pre-existing module `{}` from {:?} to {}", + module.name, + source_file, + obj_out.display()); + match link_or_copy(&source_file, &obj_out) { + Ok(_) => { } + Err(err) => { + let diag_handler = cgcx.create_diag_handler(); + diag_handler.err(&format!("unable to copy {} to {}: {}", + source_file.display(), + obj_out.display(), + err)); } } - assert_eq!(object.is_some(), config.emit_obj); - assert_eq!(bytecode.is_some(), config.emit_bc); - assert_eq!(bytecode_compressed.is_some(), config.emit_bc_compressed); - - Ok(WorkItemResult::Compiled(CompiledModule { - name: module_name, - kind: ModuleKind::Regular, - pre_existing: true, - object, - bytecode, - bytecode_compressed, - })) - } else { - debug!("llvm-optimizing {:?}", module_name); + } + + assert_eq!(object.is_some(), config.emit_obj); + assert_eq!(bytecode.is_some(), config.emit_bc); + assert_eq!(bytecode_compressed.is_some(), config.emit_bc_compressed); + + Ok(WorkItemResult::Compiled(CompiledModule { + name: module.name, + kind: ModuleKind::Regular, + object, + bytecode, + bytecode_compressed, + })) +} + +fn execute_lto_work_item(cgcx: &CodegenContext, + work_item: WorkItem, + timeline: &mut Timeline) + -> Result +{ + let config = cgcx.config(work_item.module_kind()); + + if let WorkItem::LTO(mut lto) = work_item { + let diag_handler = cgcx.create_diag_handler(); unsafe { - optimize(cgcx, &diag_handler, &module, config, timeline)?; - - let linker_does_lto = cgcx.opts.debugging_opts.cross_lang_lto.enabled(); - - // After we've done the initial round of optimizations we need to - // decide whether to synchronously codegen this module or ship it - // back to the coordinator thread for further LTO processing (which - // has to wait for all the initial modules to be optimized). - // - // Here we dispatch based on the `cgcx.lto` and kind of module we're - // codegenning... - let needs_lto = match cgcx.lto { - Lto::No => false, - - // If the linker does LTO, we don't have to do it. Note that we - // keep doing full LTO, if it is requested, as not to break the - // assumption that the output will be a single module. - Lto::Thin | Lto::ThinLocal if linker_does_lto => false, - - // Here we've got a full crate graph LTO requested. We ignore - // this, however, if the crate type is only an rlib as there's - // no full crate graph to process, that'll happen later. - // - // This use case currently comes up primarily for targets that - // require LTO so the request for LTO is always unconditionally - // passed down to the backend, but we don't actually want to do - // anything about it yet until we've got a final product. - Lto::Yes | Lto::Fat | Lto::Thin => { - cgcx.crate_types.len() != 1 || - cgcx.crate_types[0] != config::CrateType::Rlib - } + let module = lto.optimize(cgcx, timeline)?; + let module = codegen(cgcx, &diag_handler, module, config, timeline)?; + Ok(WorkItemResult::Compiled(module)) + } + } else { + bug!("execute_lto_work_item() called with wrong WorkItem kind.") + } +} - // When we're automatically doing ThinLTO for multi-codegen-unit - // builds we don't actually want to LTO the allocator modules if - // it shows up. This is due to various linker shenanigans that - // we'll encounter later. - // - // Additionally here's where we also factor in the current LLVM - // version. If it doesn't support ThinLTO we skip this. - Lto::ThinLocal => { - module.kind != ModuleKind::Allocator && - llvm::LLVMRustThinLTOAvailable() - } - }; +fn execute_load_pre_lto_mod_work_item(cgcx: &CodegenContext, + work_item: WorkItem, + _: &mut Timeline) + -> Result +{ + let module = if let WorkItem::LoadPreLtoModule(module) = work_item { + module + } else { + bug!("execute_load_pre_lto_mod_work_item() called with wrong WorkItem kind.") + }; - // Metadata modules never participate in LTO regardless of the lto - // settings. - let needs_lto = needs_lto && module.kind != ModuleKind::Metadata; + let work_product = module.source.clone(); + let incr_comp_session_dir = cgcx.incr_comp_session_dir + .as_ref() + .unwrap(); - if needs_lto { - Ok(WorkItemResult::NeedsLTO(module)) - } else { - let module = codegen(cgcx, &diag_handler, module, config, timeline)?; - Ok(WorkItemResult::Compiled(module)) + let filename = pre_lto_bitcode_filename(&work_product); + let bc_path = in_incr_comp_dir(&incr_comp_session_dir, &filename); + + let file = fs::File::open(&bc_path).unwrap_or_else(|e| { + panic!("failed to open bitcode file `{}`: {}", + bc_path.display(), + e); + }); + + let module_llvm = unsafe { + let data = ::memmap::Mmap::map(&file).unwrap_or_else(|e| { + panic!("failed to create mmap for bitcode file `{}`: {}", + bc_path.display(), + e); + }); + + let llcx = llvm::LLVMRustContextCreate(cgcx.fewer_names); + let mod_name_c = SmallCStr::new(&module.name); + let llmod_raw = match llvm::LLVMRustParseBitcodeForThinLTO( + llcx, + data.as_ptr(), + data.len(), + mod_name_c.as_ptr(), + ) { + Some(m) => m as *const _, + None => { + panic!("failed to parse bitcode for thin LTO module `{}`", + module.name); } + }; + + let tm = (cgcx.tm_factory)().unwrap(); + + ModuleLlvm { + llmod_raw, + llcx, + tm, } - } + }; + + Ok(WorkItemResult::NeedsLTO(ModuleCodegen { + name: module.name.to_string(), + module_llvm, + kind: ModuleKind::Regular, + })) } enum Message { @@ -1424,6 +1586,10 @@ enum Message { llvm_work_item: WorkItem, cost: u64, }, + AddImportOnlyModule { + module_data: SerializedModule, + module_name: CString, + }, CodegenComplete, CodegenItem, } @@ -1703,6 +1869,7 @@ fn start_executing_work(tcx: TyCtxt, let mut compiled_metadata_module = None; let mut compiled_allocator_module = None; let mut needs_lto = Vec::new(); + let mut lto_import_only_modules = Vec::new(); let mut started_lto = false; // This flag tracks whether all items have gone through codegens @@ -1749,7 +1916,7 @@ fn start_executing_work(tcx: TyCtxt, worker: get_worker_id(&mut free_worker_ids), .. cgcx.clone() }; - maybe_start_llvm_timer(cgcx.config(item.kind()), + maybe_start_llvm_timer(cgcx.config(item.module_kind()), &mut llvm_start_time); main_thread_worker_state = MainThreadWorkerState::LLVMing; spawn_work(cgcx, item); @@ -1768,7 +1935,9 @@ fn start_executing_work(tcx: TyCtxt, assert!(needs_lto.len() > 0); started_lto = true; let modules = mem::replace(&mut needs_lto, Vec::new()); - for (work, cost) in generate_lto_work(&cgcx, modules) { + let import_only_modules = + mem::replace(&mut lto_import_only_modules, Vec::new()); + for (work, cost) in generate_lto_work(&cgcx, modules, import_only_modules) { let insertion_index = work_items .binary_search_by_key(&cost, |&(_, cost)| cost) .unwrap_or_else(|e| e); @@ -1789,7 +1958,7 @@ fn start_executing_work(tcx: TyCtxt, worker: get_worker_id(&mut free_worker_ids), .. cgcx.clone() }; - maybe_start_llvm_timer(cgcx.config(item.kind()), + maybe_start_llvm_timer(cgcx.config(item.module_kind()), &mut llvm_start_time); main_thread_worker_state = MainThreadWorkerState::LLVMing; spawn_work(cgcx, item); @@ -1820,7 +1989,7 @@ fn start_executing_work(tcx: TyCtxt, while work_items.len() > 0 && running < tokens.len() { let (item, _) = work_items.pop().unwrap(); - maybe_start_llvm_timer(cgcx.config(item.kind()), + maybe_start_llvm_timer(cgcx.config(item.module_kind()), &mut llvm_start_time); let cgcx = CodegenContext { @@ -1932,10 +2101,14 @@ fn start_executing_work(tcx: TyCtxt, } else { running -= 1; } - free_worker_ids.push(worker_id); needs_lto.push(result); } + Message::AddImportOnlyModule { module_data, module_name } => { + assert!(!started_lto); + assert!(!codegen_done); + lto_import_only_modules.push((module_data, module_name)); + } Message::Done { result: Err(()), worker_id: _ } => { shared_emitter.fatal("aborting due to worker thread failure"); // Exit the coordinator thread @@ -2308,9 +2481,10 @@ impl OngoingCodegen { time_graph.dump(&format!("{}-timings", self.crate_name)); } - let work_products = copy_all_cgu_workproducts_to_incr_comp_cache_dir(sess, - &compiled_modules); - + let work_products = + copy_all_cgu_workproducts_to_incr_comp_cache_dir(sess, + &compiled_modules, + &self.output_filenames); produce_final_output_artifacts(sess, &compiled_modules, &self.output_filenames); @@ -2371,8 +2545,8 @@ impl OngoingCodegen { } pub(crate) fn submit_codegened_module_to_llvm(tcx: TyCtxt, - module: ModuleCodegen, - cost: u64) { + module: ModuleCodegen, + cost: u64) { let llvm_work_item = WorkItem::Optimize(module); drop(tcx.tx_to_llvm_workers.lock().send(Box::new(Message::CodegenDone { llvm_work_item, @@ -2380,6 +2554,61 @@ pub(crate) fn submit_codegened_module_to_llvm(tcx: TyCtxt, }))); } +pub(crate) fn submit_post_lto_module_to_llvm(tcx: TyCtxt, + module: CachedModuleCodegen) { + let llvm_work_item = WorkItem::CopyPostLtoArtifacts(module); + drop(tcx.tx_to_llvm_workers.lock().send(Box::new(Message::CodegenDone { + llvm_work_item, + cost: 0, + }))); +} + +pub(crate) fn submit_pre_lto_module_to_llvm(tcx: TyCtxt, + module: CachedModuleCodegen) { + let llvm_work_item = WorkItem::LoadPreLtoModule(module); + + drop(tcx.tx_to_llvm_workers.lock().send(Box::new(Message::CodegenDone { + llvm_work_item, + // We don't know the size of the module, but just loading will have smaller + // cost than optimizing. + cost: 10, + }))); +} + +pub(crate) fn submit_import_only_module_to_llvm(tcx: TyCtxt, + module: CachedModuleCodegen) { + let filename = pre_lto_bitcode_filename(&module.source); + let bc_path = in_incr_comp_dir_sess(tcx.sess, &filename); + let file = fs::File::open(&bc_path).unwrap_or_else(|e| { + panic!("failed to open bitcode file `{}`: {}", bc_path.display(), e) + }); + + let mmap = unsafe { + memmap::Mmap::map(&file).unwrap_or_else(|e| { + panic!("failed to mmap bitcode file `{}`: {}", bc_path.display(), e) + }) + }; + + // Schedule the module to be loaded + drop(tcx.tx_to_llvm_workers.lock().send(Box::new(Message::AddImportOnlyModule { + module_data: SerializedModule::FromUncompressedFile(mmap, file), + module_name: CString::new(module.name.clone()).unwrap(), + }))); + + // Note: We also schedule for the cached files to be copied to the output + // directory + submit_post_lto_module_to_llvm(tcx, module); +} + +fn pre_lto_bitcode_filename(wp: &WorkProduct) -> String { + wp.saved_files + .iter() + .find(|&&(kind, _)| kind == WorkProductFileKind::PreThinLtoBytecode) + .map(|&(_, ref filename)| filename.clone()) + .unwrap_or_else(|| panic!("Couldn't find pre-thin-lto bytecode for `{}`", + wp.cgu_name)) +} + fn msvc_imps_needed(tcx: TyCtxt) -> bool { // This should never be true (because it's not supported). If it is true, // something is wrong with commandline arg validation. diff --git a/src/librustc_codegen_llvm/base.rs b/src/librustc_codegen_llvm/base.rs index 59058579f643e..10e415190b4a5 100644 --- a/src/librustc_codegen_llvm/base.rs +++ b/src/librustc_codegen_llvm/base.rs @@ -24,9 +24,9 @@ //! int) and rec(x=int, y=int, z=int) will have the same llvm::Type. use super::ModuleLlvm; -use super::ModuleSource; use super::ModuleCodegen; use super::ModuleKind; +use super::CachedModuleCodegen; use abi; use back::lto; @@ -41,7 +41,6 @@ use rustc::middle::cstore::{EncodedMetadata}; use rustc::ty::{self, Ty, TyCtxt}; use rustc::ty::layout::{self, Align, TyLayout, LayoutOf}; use rustc::ty::query::Providers; -use rustc::dep_graph::{DepNode, DepConstructor}; use rustc::middle::cstore::{self, LinkagePreference}; use rustc::middle::exported_symbols; use rustc::util::common::{time, print_time_passes_entry}; @@ -699,6 +698,79 @@ pub fn iter_globals(llmod: &'ll llvm::Module) -> ValueIter<'ll> { } } +#[derive(Debug, PartialEq)] +enum CguReUsable { + No, + PreThinLto, + PostThinLto, + PostThinLtoButImportedFrom, +} + +fn determine_cgu_reuse<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, + codegen_units: &[Arc>]) + -> FxHashMap { + if !tcx.dep_graph.is_fully_enabled() { + return codegen_units.iter() + .map(|cgu| (cgu.name().clone(), CguReUsable::No)) + .collect(); + } + + let thin_lto_imports = load_thin_lto_imports(tcx.sess); + + let mut reusable_cgus = FxHashMap(); + let mut green_cgus = FxHashMap(); + let mut need_for_importing = FxHashSet(); + + for cgu in codegen_units { + let work_product_id = &cgu.work_product_id(); + if tcx.dep_graph.previous_work_product(work_product_id).is_none() { + // We don't have anything cached for this CGU. This can happen + // if the CGU did not exist in the previous session. + reusable_cgus.insert(cgu.name().clone(), CguReUsable::No); + continue + }; + // Try to mark the CGU as green + let dep_node = cgu.codegen_dep_node(tcx); + assert!(!tcx.dep_graph.dep_node_exists(&dep_node), + "CompileCodegenUnit dep-node for CGU `{}` already exists before marking.", + cgu.name()); + + if tcx.dep_graph.try_mark_green(tcx, &dep_node).is_some() { + // We can re-use either the pre- or the post-thinlto state + green_cgus.insert(cgu.name().to_string(), cgu); + } else { + // We definitely cannot re-use this CGU + reusable_cgus.insert(cgu.name().clone(), CguReUsable::No); + + let imported_cgus = thin_lto_imports.modules_imported_by(&cgu.name().as_str()); + need_for_importing.extend(imported_cgus.iter().cloned()); + } + } + + // Now we know all CGUs that have not changed themselves. Next we need to + // check if anything they imported via ThinLTO has changed. + for (cgu_name, cgu) in &green_cgus { + let imported_cgus = thin_lto_imports.modules_imported_by(cgu_name); + let all_imports_green = imported_cgus.iter().all(|imported_cgu| { + green_cgus.contains_key(&imported_cgu[..]) + }); + if all_imports_green { + reusable_cgus.insert(cgu.name().clone(), CguReUsable::PostThinLto); + } else { + reusable_cgus.insert(cgu.name().clone(), CguReUsable::PreThinLto); + need_for_importing.extend(imported_cgus.iter().cloned()); + } + } + + for (name, state) in reusable_cgus.iter_mut() { + if *state == CguReUsable::PostThinLto && need_for_importing.contains(&name.as_str()[..]) { + *state = CguReUsable::PostThinLtoButImportedFrom; + } + } + + reusable_cgus +} + pub fn codegen_crate<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, rx: mpsc::Receiver>) -> OngoingCodegen { @@ -735,7 +807,7 @@ pub fn codegen_crate<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, let metadata_module = ModuleCodegen { name: metadata_cgu_name, - source: ModuleSource::Codegened(metadata_llvm_module), + module_llvm: metadata_llvm_module, kind: ModuleKind::Metadata, }; @@ -824,7 +896,7 @@ pub fn codegen_crate<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, Some(ModuleCodegen { name: llmod_id, - source: ModuleSource::Codegened(modules), + module_llvm: modules, kind: ModuleKind::Allocator, }) } else { @@ -848,52 +920,53 @@ pub fn codegen_crate<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, let mut total_codegen_time = Duration::new(0, 0); let mut all_stats = Stats::default(); + let cgu_reuse = determine_cgu_reuse(tcx, &codegen_units); + for cgu in codegen_units.into_iter() { ongoing_codegen.wait_for_signal_to_codegen_item(); ongoing_codegen.check_for_errors(tcx.sess); - // First, if incremental compilation is enabled, we try to re-use the - // codegen unit from the cache. - if tcx.dep_graph.is_fully_enabled() { - let cgu_id = cgu.work_product_id(); - - // Check whether there is a previous work-product we can - // re-use. Not only must the file exist, and the inputs not - // be dirty, but the hash of the symbols we will generate must - // be the same. - if let Some(buf) = tcx.dep_graph.previous_work_product(&cgu_id) { - let dep_node = &DepNode::new(tcx, - DepConstructor::CompileCodegenUnit(cgu.name().clone())); - - // We try to mark the DepNode::CompileCodegenUnit green. If we - // succeed it means that none of the dependencies has changed - // and we can safely re-use. - if let Some(dep_node_index) = tcx.dep_graph.try_mark_green(tcx, dep_node) { - let module = ModuleCodegen { - name: cgu.name().to_string(), - source: ModuleSource::Preexisting(buf), - kind: ModuleKind::Regular, - }; - tcx.dep_graph.mark_loaded_from_cache(dep_node_index, true); - write::submit_codegened_module_to_llvm(tcx, module, 0); - // Continue to next cgu, this one is done. - continue - } - } else { - // This can happen if files were deleted from the cache - // directory for some reason. We just re-compile then. + let loaded_from_cache = match cgu_reuse[cgu.name()] { + CguReUsable::No => { + let _timing_guard = time_graph.as_ref().map(|time_graph| { + time_graph.start(write::CODEGEN_WORKER_TIMELINE, + write::CODEGEN_WORK_PACKAGE_KIND, + &format!("codegen {}", cgu.name())) + }); + let start_time = Instant::now(); + let stats = compile_codegen_unit(tcx, *cgu.name()); + all_stats.extend(stats); + total_codegen_time += start_time.elapsed(); + false } - } + CguReUsable::PreThinLto => { + write::submit_pre_lto_module_to_llvm(tcx, CachedModuleCodegen { + name: cgu.name().to_string(), + source: cgu.work_product(tcx), + }); + true + } + CguReUsable::PostThinLtoButImportedFrom => { + write::submit_import_only_module_to_llvm(tcx, CachedModuleCodegen { + name: cgu.name().to_string(), + source: cgu.work_product(tcx), + }); + true + } + CguReUsable::PostThinLto => { + write::submit_post_lto_module_to_llvm(tcx, CachedModuleCodegen { + name: cgu.name().to_string(), + source: cgu.work_product(tcx), + }); + true + } + }; - let _timing_guard = time_graph.as_ref().map(|time_graph| { - time_graph.start(write::CODEGEN_WORKER_TIMELINE, - write::CODEGEN_WORK_PACKAGE_KIND, - &format!("codegen {}", cgu.name())) - }); - let start_time = Instant::now(); - all_stats.extend(compile_codegen_unit(tcx, *cgu.name())); - total_codegen_time += start_time.elapsed(); - ongoing_codegen.check_for_errors(tcx.sess); + if tcx.dep_graph.is_fully_enabled() { + let dep_node = cgu.codegen_dep_node(tcx); + let dep_node_index = tcx.dep_graph.dep_node_index_of(&dep_node); + tcx.dep_graph.mark_loaded_from_cache(dep_node_index, loaded_from_cache); + } } ongoing_codegen.codegen_finished(tcx); @@ -1176,12 +1249,6 @@ fn compile_codegen_unit<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, write::submit_codegened_module_to_llvm(tcx, module, cost); - - if tcx.dep_graph.is_fully_enabled() { - let dep_node_index = tcx.dep_graph.dep_node_index_of(&dep_node); - tcx.dep_graph.mark_loaded_from_cache(dep_node_index, false); - } - return stats; fn module_codegen<'a, 'tcx>( @@ -1246,7 +1313,7 @@ fn compile_codegen_unit<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, (stats, ModuleCodegen { name: cgu_name.to_string(), - source: ModuleSource::Codegened(llvm_module), + module_llvm: llvm_module, kind: ModuleKind::Regular, }) } @@ -1324,8 +1391,11 @@ pub fn visibility_to_llvm(linkage: Visibility) -> llvm::Visibility { } } -#[allow(unused)] fn load_thin_lto_imports(sess: &Session) -> lto::ThinLTOImports { + if sess.opts.incremental.is_none() { + return lto::ThinLTOImports::new(); + } + let path = rustc_incremental::in_incr_comp_dir_sess( sess, lto::THIN_LTO_IMPORTS_INCR_COMP_FILE_NAME diff --git a/src/librustc_codegen_llvm/lib.rs b/src/librustc_codegen_llvm/lib.rs index e02fd2cae2036..dcdd8c1f6e9f9 100644 --- a/src/librustc_codegen_llvm/lib.rs +++ b/src/librustc_codegen_llvm/lib.rs @@ -66,13 +66,13 @@ extern crate rustc_errors as errors; extern crate serialize; extern crate cc; // Used to locate MSVC extern crate tempfile; +extern crate memmap; use back::bytecode::RLIB_BYTECODE_EXTENSION; pub use llvm_util::target_features; - use std::any::Any; -use std::path::PathBuf; +use std::path::{PathBuf}; use std::sync::mpsc; use rustc_data_structures::sync::Lrc; @@ -273,10 +273,15 @@ struct ModuleCodegen { /// as the crate name and disambiguator. /// We currently generate these names via CodegenUnit::build_cgu_name(). name: String, - source: ModuleSource, + module_llvm: ModuleLlvm, kind: ModuleKind, } +struct CachedModuleCodegen { + name: String, + source: WorkProduct, +} + #[derive(Copy, Clone, Debug, PartialEq)] enum ModuleKind { Regular, @@ -285,22 +290,11 @@ enum ModuleKind { } impl ModuleCodegen { - fn llvm(&self) -> Option<&ModuleLlvm> { - match self.source { - ModuleSource::Codegened(ref llvm) => Some(llvm), - ModuleSource::Preexisting(_) => None, - } - } - fn into_compiled_module(self, - emit_obj: bool, - emit_bc: bool, - emit_bc_compressed: bool, - outputs: &OutputFilenames) -> CompiledModule { - let pre_existing = match self.source { - ModuleSource::Preexisting(_) => true, - ModuleSource::Codegened(_) => false, - }; + emit_obj: bool, + emit_bc: bool, + emit_bc_compressed: bool, + outputs: &OutputFilenames) -> CompiledModule { let object = if emit_obj { Some(outputs.temp_path(OutputType::Object, Some(&self.name))) } else { @@ -321,7 +315,6 @@ impl ModuleCodegen { CompiledModule { name: self.name.clone(), kind: self.kind, - pre_existing, object, bytecode, bytecode_compressed, @@ -333,20 +326,11 @@ impl ModuleCodegen { struct CompiledModule { name: String, kind: ModuleKind, - pre_existing: bool, object: Option, bytecode: Option, bytecode_compressed: Option, } -enum ModuleSource { - /// Copy the `.o` files or whatever from the incr. comp. directory. - Preexisting(WorkProduct), - - /// Rebuild from this LLVM module. - Codegened(ModuleLlvm), -} - struct ModuleLlvm { llcx: &'static mut llvm::Context, llmod_raw: *const llvm::Module, diff --git a/src/librustc_incremental/persist/work_product.rs b/src/librustc_incremental/persist/work_product.rs index cfe59b1f67262..c285934d75bb9 100644 --- a/src/librustc_incremental/persist/work_product.rs +++ b/src/librustc_incremental/persist/work_product.rs @@ -36,6 +36,7 @@ pub fn copy_cgu_workproducts_to_incr_comp_cache_dir( WorkProductFileKind::Object => "o", WorkProductFileKind::Bytecode => "bc", WorkProductFileKind::BytecodeCompressed => "bc.z", + WorkProductFileKind::PreThinLtoBytecode => "pre-thinlto.bc", }; let file_name = format!("{}.{}", cgu_name, extension); let path_in_incr_dir = in_incr_comp_dir_sess(sess, &file_name); diff --git a/src/librustc_mir/monomorphize/partitioning.rs b/src/librustc_mir/monomorphize/partitioning.rs index a8c27c037513f..fd094ffc1cf4f 100644 --- a/src/librustc_mir/monomorphize/partitioning.rs +++ b/src/librustc_mir/monomorphize/partitioning.rs @@ -103,7 +103,7 @@ //! inlining, even when they are not marked #[inline]. use monomorphize::collector::InliningMap; -use rustc::dep_graph::{WorkProductId, DepNode, DepConstructor}; +use rustc::dep_graph::{WorkProductId, WorkProduct, DepNode, DepConstructor}; use rustc::hir::CodegenFnAttrFlags; use rustc::hir::def_id::{DefId, LOCAL_CRATE, CRATE_DEF_INDEX}; use rustc::hir::map::DefPathData; @@ -150,6 +150,15 @@ pub trait CodegenUnitExt<'tcx> { WorkProductId::from_cgu_name(&self.name().as_str()) } + fn work_product(&self, tcx: TyCtxt) -> WorkProduct { + let work_product_id = self.work_product_id(); + tcx.dep_graph + .previous_work_product(&work_product_id) + .unwrap_or_else(|| { + panic!("Could not find work-product for CGU `{}`", self.name()) + }) + } + fn items_in_deterministic_order<'a>(&self, tcx: TyCtxt<'a, 'tcx, 'tcx>) -> Vec<(MonoItem<'tcx>, From abd5cc3364b9ec9500ef8f5e64699e25069bc51b Mon Sep 17 00:00:00 2001 From: Michael Woerister Date: Fri, 31 Aug 2018 15:18:08 +0200 Subject: [PATCH 5/7] Always add all modules to the global ThinLTO module analysis when compiling incrementally. --- src/Cargo.lock | 11 + src/librustc/dep_graph/graph.rs | 1 - src/librustc_codegen_llvm/back/lto.rs | 235 ++++++++---------- src/librustc_codegen_llvm/back/write.rs | 154 +++--------- src/librustc_codegen_llvm/base.rs | 129 +++------- .../persist/work_product.rs | 1 - 6 files changed, 170 insertions(+), 361 deletions(-) diff --git a/src/Cargo.lock b/src/Cargo.lock index e4d9f6e23945e..defb5b9869d82 100644 --- a/src/Cargo.lock +++ b/src/Cargo.lock @@ -1198,6 +1198,15 @@ dependencies = [ "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "memmap" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "memoffset" version = "0.2.1" @@ -2029,6 +2038,7 @@ name = "rustc_codegen_llvm" version = "0.0.0" dependencies = [ "cc 1.0.22 (registry+https://github.com/rust-lang/crates.io-index)", + "memmap 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", "num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-demangle 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", "rustc_llvm 0.0.0", @@ -3151,6 +3161,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" "checksum mdbook 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "90b5a8d7e341ceee5db3882a06078d42661ddcfa2b3687319cc5da76ec4e782f" "checksum memchr 2.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a3b4142ab8738a78c51896f704f83c11df047ff1bda9a92a661aa6361552d93d" +"checksum memmap 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e2ffa2c986de11a9df78620c01eeaaf27d94d3ff02bf81bfcca953102dd0c6ff" "checksum memoffset 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0f9dc261e2b62d7a622bf416ea3c5245cdd5d9a7fcc428c0d06804dfce1775b3" "checksum minifier 0.0.19 (registry+https://github.com/rust-lang/crates.io-index)" = "9908ed7c62f990c21ab41fdca53a864a3ada0da69d8729c4de727b397e27bc11" "checksum miniz-sys 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "609ce024854aeb19a0ef7567d348aaa5a746b32fb72e336df7fcc16869d7e2b4" diff --git a/src/librustc/dep_graph/graph.rs b/src/librustc/dep_graph/graph.rs index 1c883eeed966c..4df0fc443a27c 100644 --- a/src/librustc/dep_graph/graph.rs +++ b/src/librustc/dep_graph/graph.rs @@ -883,7 +883,6 @@ pub enum WorkProductFileKind { Object, Bytecode, BytecodeCompressed, - PreThinLtoBytecode, } pub(super) struct CurrentDepGraph { diff --git a/src/librustc_codegen_llvm/back/lto.rs b/src/librustc_codegen_llvm/back/lto.rs index 9813134992760..6936a3067800d 100644 --- a/src/librustc_codegen_llvm/back/lto.rs +++ b/src/librustc_codegen_llvm/back/lto.rs @@ -11,17 +11,18 @@ use back::bytecode::{DecodedBytecode, RLIB_BYTECODE_EXTENSION}; use back::symbol_export; use back::write::{ModuleConfig, with_llvm_pmb, CodegenContext}; -use back::write::{self, DiagnosticHandlers}; +use back::write::{self, DiagnosticHandlers, pre_lto_bitcode_filename}; use errors::{FatalError, Handler}; use llvm::archive_ro::ArchiveRO; use llvm::{True, False}; use llvm; use memmap; +use rustc::dep_graph::WorkProduct; use rustc::hir::def_id::LOCAL_CRATE; use rustc::middle::exported_symbols::SymbolExportLevel; use rustc::session::config::{self, Lto}; use rustc::util::common::time_ext; -use rustc_data_structures::fx::{FxHashMap, FxHashSet}; +use rustc_data_structures::fx::FxHashMap; use time_graph::Timeline; use {ModuleCodegen, ModuleLlvm, ModuleKind}; @@ -29,15 +30,10 @@ use libc; use std::ffi::{CStr, CString}; use std::fs::File; -use std::io; -use std::mem; -use std::path::Path; use std::ptr; use std::slice; use std::sync::Arc; -pub const THIN_LTO_IMPORTS_INCR_COMP_FILE_NAME: &str = "thin-lto-imports.bin"; - pub fn crate_type_allows_lto(crate_type: config::CrateType) -> bool { match crate_type { config::CrateType::Executable | @@ -105,11 +101,16 @@ impl LtoModuleCodegen { } } +/// Performs LTO, which in the case of full LTO means merging all modules into +/// a single one and returning it for further optimizing. For ThinLTO, it will +/// do the global analysis necessary and return two lists, one of the modules +/// the need optimization and another for modules that can simply be copied over +/// from the incr. comp. cache. pub(crate) fn run(cgcx: &CodegenContext, modules: Vec, - import_only_modules: Vec<(SerializedModule, CString)>, + cached_modules: Vec<(SerializedModule, WorkProduct)>, timeline: &mut Timeline) - -> Result, FatalError> + -> Result<(Vec, Vec), FatalError> { let diag_handler = cgcx.create_diag_handler(); let export_threshold = match cgcx.lto { @@ -202,13 +203,14 @@ pub(crate) fn run(cgcx: &CodegenContext, match cgcx.lto { Lto::Yes | // `-C lto` == fat LTO by default Lto::Fat => { - assert!(import_only_modules.is_empty()); - fat_lto(cgcx, - &diag_handler, - modules, - upstream_modules, - &symbol_white_list, - timeline) + assert!(cached_modules.is_empty()); + let opt_jobs = fat_lto(cgcx, + &diag_handler, + modules, + upstream_modules, + &symbol_white_list, + timeline); + opt_jobs.map(|opt_jobs| (opt_jobs, vec![])) } Lto::Thin | Lto::ThinLocal => { @@ -220,7 +222,7 @@ pub(crate) fn run(cgcx: &CodegenContext, &diag_handler, modules, upstream_modules, - import_only_modules, + cached_modules, &symbol_white_list, timeline) } @@ -388,14 +390,19 @@ fn thin_lto(cgcx: &CodegenContext, diag_handler: &Handler, modules: Vec, serialized_modules: Vec<(SerializedModule, CString)>, - import_only_modules: Vec<(SerializedModule, CString)>, + cached_modules: Vec<(SerializedModule, WorkProduct)>, symbol_white_list: &[*const libc::c_char], timeline: &mut Timeline) - -> Result, FatalError> + -> Result<(Vec, Vec), FatalError> { unsafe { info!("going for that thin, thin LTO"); + let green_modules: FxHashMap<_, _> = cached_modules + .iter() + .map(|&(_, ref wp)| (wp.cgu_name.clone(), wp.clone())) + .collect(); + let mut thin_buffers = Vec::new(); let mut module_names = Vec::new(); let mut thin_modules = Vec::new(); @@ -411,6 +418,28 @@ fn thin_lto(cgcx: &CodegenContext, info!("local module: {} - {}", i, module.name); let name = CString::new(module.name.clone()).unwrap(); let buffer = ThinBuffer::new(module.module_llvm.llmod()); + + // We emit the module after having serialized it into a ThinBuffer + // because only then it will contain the ThinLTO module summary. + if let Some(ref incr_comp_session_dir) = cgcx.incr_comp_session_dir { + if cgcx.config(module.kind).emit_pre_thin_lto_bc { + use std::io::Write; + + let path = incr_comp_session_dir + .join(pre_lto_bitcode_filename(&module.name)); + let mut file = File::create(&path).unwrap_or_else(|e| { + panic!("Failed to create pre-lto-bitcode file `{}`: {}", + path.display(), + e); + }); + file.write_all(buffer.data()).unwrap_or_else(|e| { + panic!("Error writing pre-lto-bitcode file `{}`: {}", + path.display(), + e); + }); + } + } + thin_modules.push(llvm::ThinLTOModule { identifier: name.as_ptr(), data: buffer.data().as_ptr(), @@ -438,24 +467,13 @@ fn thin_lto(cgcx: &CodegenContext, // looking at upstream modules entirely sometimes (the contents, // we must always unconditionally look at the index). let mut serialized = Vec::new(); - for (module, name) in serialized_modules { - info!("foreign module {:?}", name); - thin_modules.push(llvm::ThinLTOModule { - identifier: name.as_ptr(), - data: module.data().as_ptr(), - len: module.data().len(), - }); - serialized.push(module); - module_names.push(name); - } - // All the modules collected up to this point we actually want to - // optimize. The `import_only_modules` below need to be in the list of - // available modules but we don't need to run optimizations for them - // since we already have their optimized version cached. - let modules_to_optimize = module_names.len(); - for (module, name) in import_only_modules { - info!("foreign module {:?}", name); + let cached_modules = cached_modules.into_iter().map(|(sm, wp)| { + (sm, CString::new(wp.cgu_name).unwrap()) + }); + + for (module, name) in serialized_modules.into_iter().chain(cached_modules) { + info!("upstream or cached module {:?}", name); thin_modules.push(llvm::ThinLTOModule { identifier: name.as_ptr(), data: module.data().as_ptr(), @@ -465,6 +483,9 @@ fn thin_lto(cgcx: &CodegenContext, module_names.push(name); } + // Sanity check + assert_eq!(thin_modules.len(), module_names.len()); + // Delegate to the C++ bindings to create some data here. Once this is a // tried-and-true interface we may wish to try to upstream some of this // to LLVM itself, right now we reimplement a lot of what they do @@ -478,30 +499,7 @@ fn thin_lto(cgcx: &CodegenContext, write::llvm_err(&diag_handler, "failed to prepare thin LTO context".to_string()) })?; - // Save the ThinLTO import information for incremental compilation. - if let Some(ref incr_comp_session_dir) = cgcx.incr_comp_session_dir { - let path = incr_comp_session_dir.join(THIN_LTO_IMPORTS_INCR_COMP_FILE_NAME); - - // The import information from the current compilation session. It - // does not contain info about modules that have been loaded from - // the cache instead of having been recompiled... - let current_imports = ThinLTOImports::from_thin_lto_data(data); - - // ... so we load this additional information from the previous - // cache file if necessary. - let imports = if path.exists() { - let prev_imports = ThinLTOImports::load_from_file(&path).unwrap(); - prev_imports.update(current_imports, &module_names) - } else { - current_imports - }; - - if let Err(err) = imports.save_to_file(&path) { - let msg = format!("Error while writing ThinLTO import data: {}", - err); - return Err(write::llvm_err(&diag_handler, msg)); - } - } + let import_map = ThinLTOImports::from_thin_lto_data(data); let data = ThinData(data); info!("thin LTO data created"); @@ -517,12 +515,36 @@ fn thin_lto(cgcx: &CodegenContext, serialized_modules: serialized, module_names, }); - Ok((0..modules_to_optimize).map(|i| { - LtoModuleCodegen::Thin(ThinModule { + + let mut copy_jobs = vec![]; + let mut opt_jobs = vec![]; + + for (module_index, module_name) in shared.module_names.iter().enumerate() { + let module_name = module_name_to_str(module_name); + + if green_modules.contains_key(module_name) { + let mut imports_all_green = true; + for imported_module in import_map.modules_imported_by(module_name) { + if !green_modules.contains_key(imported_module) { + imports_all_green = false; + break + } + } + + if imports_all_green { + let work_product = green_modules[module_name].clone(); + copy_jobs.push(work_product); + continue + } + } + + opt_jobs.push(LtoModuleCodegen::Thin(ThinModule { shared: shared.clone(), - idx: i, - }) - }).collect()) + idx: module_index, + })); + } + + Ok((opt_jobs, copy_jobs)) } } @@ -850,44 +872,12 @@ pub struct ThinLTOImports { } impl ThinLTOImports { - pub fn new() -> ThinLTOImports { - ThinLTOImports { - imports: FxHashMap(), - } - } - pub fn modules_imported_by(&self, llvm_module_name: &str) -> &[String] { self.imports.get(llvm_module_name).map(|v| &v[..]).unwrap_or(&[]) } - pub fn update(mut self, new: ThinLTOImports, module_names: &[CString]) -> ThinLTOImports { - let module_names: FxHashSet<_> = module_names.iter().map(|name| { - name.clone().into_string().unwrap() - }).collect(); - - // Remove all modules that don't exist anymore. - self.imports.retain(|k, _| module_names.contains(k)); - - // Overwrite old values - for (importing_module, imported_modules) in new.imports { - self.imports.insert(importing_module, imported_modules); - } - - self - } - /// Load the ThinLTO import map from ThinLTOData. unsafe fn from_thin_lto_data(data: *const llvm::ThinLTOData) -> ThinLTOImports { - fn module_name_to_str(c_str: &CStr) -> &str { - match c_str.to_str() { - Ok(s) => s, - Err(e) => { - bug!("Encountered non-utf8 LLVM module name `{}`: {}", - c_str.to_string_lossy(), - e) - } - } - } unsafe extern "C" fn imported_module_callback(payload: *mut libc::c_void, importing_module_name: *const libc::c_char, imported_module_name: *const libc::c_char) { @@ -896,6 +886,7 @@ impl ThinLTOImports { let importing_module_name = module_name_to_str(&importing_module_name); let imported_module_name = CStr::from_ptr(imported_module_name); let imported_module_name = module_name_to_str(&imported_module_name); + if !map.imports.contains_key(importing_module_name) { map.imports.insert(importing_module_name.to_owned(), vec![]); } @@ -913,47 +904,15 @@ impl ThinLTOImports { &mut map as *mut _ as *mut libc::c_void); map } +} - pub fn save_to_file(&self, path: &Path) -> io::Result<()> { - use std::io::Write; - let file = File::create(path)?; - let mut writer = io::BufWriter::new(file); - for (importing_module_name, imported_modules) in &self.imports { - writeln!(writer, "{}", importing_module_name)?; - for imported_module in imported_modules { - writeln!(writer, " {}", imported_module)?; - } - writeln!(writer)?; - } - Ok(()) - } - - pub fn load_from_file(path: &Path) -> io::Result { - use std::io::BufRead; - let mut imports = FxHashMap(); - let mut current_module = None; - let mut current_imports = vec![]; - let file = File::open(path)?; - for line in io::BufReader::new(file).lines() { - let line = line?; - if line.is_empty() { - let importing_module = current_module - .take() - .expect("Importing module not set"); - imports.insert(importing_module, - mem::replace(&mut current_imports, vec![])); - } else if line.starts_with(" ") { - // This is an imported module - assert_ne!(current_module, None); - current_imports.push(line.trim().to_string()); - } else { - // This is the beginning of a new module - assert_eq!(current_module, None); - current_module = Some(line.trim().to_string()); - } +fn module_name_to_str(c_str: &CStr) -> &str { + match c_str.to_str() { + Ok(s) => s, + Err(e) => { + bug!("Encountered non-utf8 LLVM module name `{}`: {}", + c_str.to_string_lossy(), + e) } - Ok(ThinLTOImports { - imports - }) } -} +} \ No newline at end of file diff --git a/src/librustc_codegen_llvm/back/write.rs b/src/librustc_codegen_llvm/back/write.rs index e1d69db83b99d..9c0f4c3979008 100644 --- a/src/librustc_codegen_llvm/back/write.rs +++ b/src/librustc_codegen_llvm/back/write.rs @@ -28,7 +28,7 @@ use rustc::util::nodemap::FxHashMap; use time_graph::{self, TimeGraph, Timeline}; use llvm::{self, DiagnosticInfo, PassManager, SMDiagnostic}; use llvm_util; -use {CodegenResults, ModuleCodegen, CompiledModule, ModuleKind, ModuleLlvm, +use {CodegenResults, ModuleCodegen, CompiledModule, ModuleKind, // ModuleLlvm, CachedModuleCodegen}; use CrateInfo; use rustc::hir::def_id::{CrateNum, LOCAL_CRATE}; @@ -228,8 +228,8 @@ pub struct ModuleConfig { pgo_use: String, // Flags indicating which outputs to produce. + pub emit_pre_thin_lto_bc: bool, emit_no_opt_bc: bool, - emit_pre_thin_lto_bc: bool, emit_bc: bool, emit_bc_compressed: bool, emit_lto_bc: bool, @@ -625,20 +625,13 @@ unsafe fn optimize(cgcx: &CodegenContext, // Deallocate managers that we're now done with llvm::LLVMDisposePassManager(fpm); llvm::LLVMDisposePassManager(mpm); - - if config.emit_pre_thin_lto_bc { - let out = cgcx.output_filenames.temp_path_ext(PRE_THIN_LTO_BC_EXT, - module_name); - let out = path2cstr(&out); - llvm::LLVMWriteBitcodeToFile(llmod, out.as_ptr()); - } } Ok(()) } fn generate_lto_work(cgcx: &CodegenContext, modules: Vec, - import_only_modules: Vec<(SerializedModule, CString)>) + import_only_modules: Vec<(SerializedModule, WorkProduct)>) -> Vec<(WorkItem, u64)> { let mut timeline = cgcx.time_graph.as_ref().map(|tg| { @@ -646,13 +639,22 @@ fn generate_lto_work(cgcx: &CodegenContext, CODEGEN_WORK_PACKAGE_KIND, "generate lto") }).unwrap_or(Timeline::noop()); - let lto_modules = lto::run(cgcx, modules, import_only_modules, &mut timeline) + let (lto_modules, copy_jobs) = lto::run(cgcx, modules, import_only_modules, &mut timeline) .unwrap_or_else(|e| e.raise()); - lto_modules.into_iter().map(|module| { + let lto_modules = lto_modules.into_iter().map(|module| { let cost = module.cost(); (WorkItem::LTO(module), cost) - }).collect() + }); + + let copy_jobs = copy_jobs.into_iter().map(|wp| { + (WorkItem::CopyPostLtoArtifacts(CachedModuleCodegen { + name: wp.cgu_name.clone(), + source: wp, + }), 0) + }); + + lto_modules.chain(copy_jobs).collect() } unsafe fn codegen(cgcx: &CodegenContext, @@ -1083,7 +1085,6 @@ pub fn start_async_codegen(tcx: TyCtxt, fn copy_all_cgu_workproducts_to_incr_comp_cache_dir( sess: &Session, compiled_modules: &CompiledModules, - output_filenames: &OutputFilenames, ) -> FxHashMap { let mut work_products = FxHashMap::default(); @@ -1104,13 +1105,6 @@ fn copy_all_cgu_workproducts_to_incr_comp_cache_dir( files.push((WorkProductFileKind::BytecodeCompressed, path.clone())); } - let pre_thin_lto_bytecode_path = - output_filenames.temp_path_ext(PRE_THIN_LTO_BC_EXT, Some(&module.name)); - - if pre_thin_lto_bytecode_path.exists() { - files.push((WorkProductFileKind::PreThinLtoBytecode, pre_thin_lto_bytecode_path)); - } - if let Some((id, product)) = copy_cgu_workproducts_to_incr_comp_cache_dir(sess, &module.name, &files) { work_products.insert(id, product); @@ -1285,9 +1279,6 @@ enum WorkItem { /// Copy the post-LTO artifacts from the incremental cache to the output /// directory. CopyPostLtoArtifacts(CachedModuleCodegen), - /// Load the pre-LTO version of a module from the incremental cache, so it - /// can be run through LTO again. - LoadPreLtoModule(CachedModuleCodegen), /// Perform (Thin)LTO on the given module. LTO(lto::LtoModuleCodegen), } @@ -1297,7 +1288,6 @@ impl WorkItem { match *self { WorkItem::Optimize(ref m) => m.kind, WorkItem::CopyPostLtoArtifacts(_) | - WorkItem::LoadPreLtoModule(_) | WorkItem::LTO(_) => ModuleKind::Regular, } } @@ -1305,7 +1295,6 @@ impl WorkItem { fn name(&self) -> String { match *self { WorkItem::Optimize(ref m) => format!("optimize: {}", m.name), - WorkItem::LoadPreLtoModule(ref m) => format!("load pre-lto module: {}", m.name), WorkItem::CopyPostLtoArtifacts(ref m) => format!("copy post LTO artifacts: {}", m.name), WorkItem::LTO(ref m) => format!("lto: {}", m.name()), } @@ -1326,9 +1315,6 @@ fn execute_work_item(cgcx: &CodegenContext, work_item @ WorkItem::Optimize(_) => { execute_optimize_work_item(cgcx, work_item, timeline) } - work_item @ WorkItem::LoadPreLtoModule(_) => { - execute_load_pre_lto_mod_work_item(cgcx, work_item, timeline) - } work_item @ WorkItem::CopyPostLtoArtifacts(_) => { execute_copy_from_cache_work_item(cgcx, work_item, timeline) } @@ -1454,9 +1440,6 @@ fn execute_copy_from_cache_work_item(cgcx: &CodegenContext, bytecode_compressed = Some(path.clone()); path } - WorkProductFileKind::PreThinLtoBytecode => { - continue; - } }; let source_file = in_incr_comp_dir(&incr_comp_session_dir, &saved_file); @@ -1509,69 +1492,6 @@ fn execute_lto_work_item(cgcx: &CodegenContext, } } -fn execute_load_pre_lto_mod_work_item(cgcx: &CodegenContext, - work_item: WorkItem, - _: &mut Timeline) - -> Result -{ - let module = if let WorkItem::LoadPreLtoModule(module) = work_item { - module - } else { - bug!("execute_load_pre_lto_mod_work_item() called with wrong WorkItem kind.") - }; - - let work_product = module.source.clone(); - let incr_comp_session_dir = cgcx.incr_comp_session_dir - .as_ref() - .unwrap(); - - let filename = pre_lto_bitcode_filename(&work_product); - let bc_path = in_incr_comp_dir(&incr_comp_session_dir, &filename); - - let file = fs::File::open(&bc_path).unwrap_or_else(|e| { - panic!("failed to open bitcode file `{}`: {}", - bc_path.display(), - e); - }); - - let module_llvm = unsafe { - let data = ::memmap::Mmap::map(&file).unwrap_or_else(|e| { - panic!("failed to create mmap for bitcode file `{}`: {}", - bc_path.display(), - e); - }); - - let llcx = llvm::LLVMRustContextCreate(cgcx.fewer_names); - let mod_name_c = SmallCStr::new(&module.name); - let llmod_raw = match llvm::LLVMRustParseBitcodeForThinLTO( - llcx, - data.as_ptr(), - data.len(), - mod_name_c.as_ptr(), - ) { - Some(m) => m as *const _, - None => { - panic!("failed to parse bitcode for thin LTO module `{}`", - module.name); - } - }; - - let tm = (cgcx.tm_factory)().unwrap(); - - ModuleLlvm { - llmod_raw, - llcx, - tm, - } - }; - - Ok(WorkItemResult::NeedsLTO(ModuleCodegen { - name: module.name.to_string(), - module_llvm, - kind: ModuleKind::Regular, - })) -} - enum Message { Token(io::Result), NeedsLTO { @@ -1588,7 +1508,7 @@ enum Message { }, AddImportOnlyModule { module_data: SerializedModule, - module_name: CString, + work_product: WorkProduct, }, CodegenComplete, CodegenItem, @@ -1893,6 +1813,7 @@ fn start_executing_work(tcx: TyCtxt, work_items.len() > 0 || running > 0 || needs_lto.len() > 0 || + lto_import_only_modules.len() > 0 || main_thread_worker_state != MainThreadWorkerState::Idle { // While there are still CGUs to be codegened, the coordinator has @@ -1932,7 +1853,7 @@ fn start_executing_work(tcx: TyCtxt, running == 0 && main_thread_worker_state == MainThreadWorkerState::Idle { assert!(!started_lto); - assert!(needs_lto.len() > 0); + assert!(needs_lto.len() + lto_import_only_modules.len() > 0); started_lto = true; let modules = mem::replace(&mut needs_lto, Vec::new()); let import_only_modules = @@ -2104,10 +2025,13 @@ fn start_executing_work(tcx: TyCtxt, free_worker_ids.push(worker_id); needs_lto.push(result); } - Message::AddImportOnlyModule { module_data, module_name } => { + Message::AddImportOnlyModule { module_data, work_product } => { assert!(!started_lto); assert!(!codegen_done); - lto_import_only_modules.push((module_data, module_name)); + assert_eq!(main_thread_worker_state, + MainThreadWorkerState::Codegenning); + lto_import_only_modules.push((module_data, work_product)); + main_thread_worker_state = MainThreadWorkerState::Idle; } Message::Done { result: Err(()), worker_id: _ } => { shared_emitter.fatal("aborting due to worker thread failure"); @@ -2483,8 +2407,7 @@ impl OngoingCodegen { let work_products = copy_all_cgu_workproducts_to_incr_comp_cache_dir(sess, - &compiled_modules, - &self.output_filenames); + &compiled_modules); produce_final_output_artifacts(sess, &compiled_modules, &self.output_filenames); @@ -2565,19 +2488,7 @@ pub(crate) fn submit_post_lto_module_to_llvm(tcx: TyCtxt, pub(crate) fn submit_pre_lto_module_to_llvm(tcx: TyCtxt, module: CachedModuleCodegen) { - let llvm_work_item = WorkItem::LoadPreLtoModule(module); - - drop(tcx.tx_to_llvm_workers.lock().send(Box::new(Message::CodegenDone { - llvm_work_item, - // We don't know the size of the module, but just loading will have smaller - // cost than optimizing. - cost: 10, - }))); -} - -pub(crate) fn submit_import_only_module_to_llvm(tcx: TyCtxt, - module: CachedModuleCodegen) { - let filename = pre_lto_bitcode_filename(&module.source); + let filename = pre_lto_bitcode_filename(&module.name); let bc_path = in_incr_comp_dir_sess(tcx.sess, &filename); let file = fs::File::open(&bc_path).unwrap_or_else(|e| { panic!("failed to open bitcode file `{}`: {}", bc_path.display(), e) @@ -2592,21 +2503,12 @@ pub(crate) fn submit_import_only_module_to_llvm(tcx: TyCtxt, // Schedule the module to be loaded drop(tcx.tx_to_llvm_workers.lock().send(Box::new(Message::AddImportOnlyModule { module_data: SerializedModule::FromUncompressedFile(mmap, file), - module_name: CString::new(module.name.clone()).unwrap(), + work_product: module.source, }))); - - // Note: We also schedule for the cached files to be copied to the output - // directory - submit_post_lto_module_to_llvm(tcx, module); } -fn pre_lto_bitcode_filename(wp: &WorkProduct) -> String { - wp.saved_files - .iter() - .find(|&&(kind, _)| kind == WorkProductFileKind::PreThinLtoBytecode) - .map(|&(_, ref filename)| filename.clone()) - .unwrap_or_else(|| panic!("Couldn't find pre-thin-lto bytecode for `{}`", - wp.cgu_name)) +pub(super) fn pre_lto_bitcode_filename(module_name: &str) -> String { + format!("{}.{}", module_name, PRE_THIN_LTO_BC_EXT) } fn msvc_imps_needed(tcx: TyCtxt) -> bool { diff --git a/src/librustc_codegen_llvm/base.rs b/src/librustc_codegen_llvm/base.rs index 10e415190b4a5..c1f6006e684be 100644 --- a/src/librustc_codegen_llvm/base.rs +++ b/src/librustc_codegen_llvm/base.rs @@ -29,7 +29,6 @@ use super::ModuleKind; use super::CachedModuleCodegen; use abi; -use back::lto; use back::write::{self, OngoingCodegen}; use llvm::{self, TypeKind, get_param}; use metadata; @@ -45,7 +44,7 @@ use rustc::middle::cstore::{self, LinkagePreference}; use rustc::middle::exported_symbols; use rustc::util::common::{time, print_time_passes_entry}; use rustc::util::profiling::ProfileCategory; -use rustc::session::config::{self, DebugInfo, EntryFnType}; +use rustc::session::config::{self, DebugInfo, EntryFnType, Lto}; use rustc::session::Session; use rustc_incremental; use allocator; @@ -698,77 +697,48 @@ pub fn iter_globals(llmod: &'ll llvm::Module) -> ValueIter<'ll> { } } -#[derive(Debug, PartialEq)] +#[derive(Debug)] enum CguReUsable { - No, - PreThinLto, - PostThinLto, - PostThinLtoButImportedFrom, + PreLto, + PostLto, + No } fn determine_cgu_reuse<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, - codegen_units: &[Arc>]) - -> FxHashMap { + cgu: &CodegenUnit<'tcx>) + -> CguReUsable { if !tcx.dep_graph.is_fully_enabled() { - return codegen_units.iter() - .map(|cgu| (cgu.name().clone(), CguReUsable::No)) - .collect(); + return CguReUsable::No } - let thin_lto_imports = load_thin_lto_imports(tcx.sess); - - let mut reusable_cgus = FxHashMap(); - let mut green_cgus = FxHashMap(); - let mut need_for_importing = FxHashSet(); - - for cgu in codegen_units { - let work_product_id = &cgu.work_product_id(); - if tcx.dep_graph.previous_work_product(work_product_id).is_none() { - // We don't have anything cached for this CGU. This can happen - // if the CGU did not exist in the previous session. - reusable_cgus.insert(cgu.name().clone(), CguReUsable::No); - continue - }; - // Try to mark the CGU as green - let dep_node = cgu.codegen_dep_node(tcx); - assert!(!tcx.dep_graph.dep_node_exists(&dep_node), - "CompileCodegenUnit dep-node for CGU `{}` already exists before marking.", - cgu.name()); - - if tcx.dep_graph.try_mark_green(tcx, &dep_node).is_some() { - // We can re-use either the pre- or the post-thinlto state - green_cgus.insert(cgu.name().to_string(), cgu); - } else { - // We definitely cannot re-use this CGU - reusable_cgus.insert(cgu.name().clone(), CguReUsable::No); - - let imported_cgus = thin_lto_imports.modules_imported_by(&cgu.name().as_str()); - need_for_importing.extend(imported_cgus.iter().cloned()); - } + let work_product_id = &cgu.work_product_id(); + if tcx.dep_graph.previous_work_product(work_product_id).is_none() { + // We don't have anything cached for this CGU. This can happen + // if the CGU did not exist in the previous session. + return CguReUsable::No } - // Now we know all CGUs that have not changed themselves. Next we need to - // check if anything they imported via ThinLTO has changed. - for (cgu_name, cgu) in &green_cgus { - let imported_cgus = thin_lto_imports.modules_imported_by(cgu_name); - let all_imports_green = imported_cgus.iter().all(|imported_cgu| { - green_cgus.contains_key(&imported_cgu[..]) - }); - if all_imports_green { - reusable_cgus.insert(cgu.name().clone(), CguReUsable::PostThinLto); + // Try to mark the CGU as green. If it we can do so, it means that nothing + // affecting the LLVM module has changed and we can re-use a cached version. + // If we compile with any kind of LTO, this means we can re-use the bitcode + // of the Pre-LTO stage (possibly also the Post-LTO version but we'll only + // know that later). If we are not doing LTO, there is only one optimized + // version of each module, so we re-use that. + let dep_node = cgu.codegen_dep_node(tcx); + assert!(!tcx.dep_graph.dep_node_exists(&dep_node), + "CompileCodegenUnit dep-node for CGU `{}` already exists before marking.", + cgu.name()); + + if tcx.dep_graph.try_mark_green(tcx, &dep_node).is_some() { + // We can re-use either the pre- or the post-thinlto state + if tcx.sess.lto() != Lto::No { + CguReUsable::PreLto } else { - reusable_cgus.insert(cgu.name().clone(), CguReUsable::PreThinLto); - need_for_importing.extend(imported_cgus.iter().cloned()); - } - } - - for (name, state) in reusable_cgus.iter_mut() { - if *state == CguReUsable::PostThinLto && need_for_importing.contains(&name.as_str()[..]) { - *state = CguReUsable::PostThinLtoButImportedFrom; + CguReUsable::PostLto } + } else { + CguReUsable::No } - - reusable_cgus } pub fn codegen_crate<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, @@ -920,13 +890,11 @@ pub fn codegen_crate<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, let mut total_codegen_time = Duration::new(0, 0); let mut all_stats = Stats::default(); - let cgu_reuse = determine_cgu_reuse(tcx, &codegen_units); - for cgu in codegen_units.into_iter() { ongoing_codegen.wait_for_signal_to_codegen_item(); ongoing_codegen.check_for_errors(tcx.sess); - let loaded_from_cache = match cgu_reuse[cgu.name()] { + let loaded_from_cache = match determine_cgu_reuse(tcx, &cgu) { CguReUsable::No => { let _timing_guard = time_graph.as_ref().map(|time_graph| { time_graph.start(write::CODEGEN_WORKER_TIMELINE, @@ -939,21 +907,14 @@ pub fn codegen_crate<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, total_codegen_time += start_time.elapsed(); false } - CguReUsable::PreThinLto => { + CguReUsable::PreLto => { write::submit_pre_lto_module_to_llvm(tcx, CachedModuleCodegen { name: cgu.name().to_string(), source: cgu.work_product(tcx), }); true } - CguReUsable::PostThinLtoButImportedFrom => { - write::submit_import_only_module_to_llvm(tcx, CachedModuleCodegen { - name: cgu.name().to_string(), - source: cgu.work_product(tcx), - }); - true - } - CguReUsable::PostThinLto => { + CguReUsable::PostLto => { write::submit_post_lto_module_to_llvm(tcx, CachedModuleCodegen { name: cgu.name().to_string(), source: cgu.work_product(tcx), @@ -1391,28 +1352,6 @@ pub fn visibility_to_llvm(linkage: Visibility) -> llvm::Visibility { } } -fn load_thin_lto_imports(sess: &Session) -> lto::ThinLTOImports { - if sess.opts.incremental.is_none() { - return lto::ThinLTOImports::new(); - } - - let path = rustc_incremental::in_incr_comp_dir_sess( - sess, - lto::THIN_LTO_IMPORTS_INCR_COMP_FILE_NAME - ); - if !path.exists() { - return lto::ThinLTOImports::new(); - } - match lto::ThinLTOImports::load_from_file(&path) { - Ok(imports) => imports, - Err(e) => { - let msg = format!("Error while trying to load ThinLTO import data \ - for incremental compilation: {}", e); - sess.fatal(&msg) - } - } -} - // FIXME(mw): Anything that is produced via DepGraph::with_task() must implement // the HashStable trait. Normally DepGraph::with_task() calls are // hidden behind queries, but CGU creation is a special case in two diff --git a/src/librustc_incremental/persist/work_product.rs b/src/librustc_incremental/persist/work_product.rs index c285934d75bb9..cfe59b1f67262 100644 --- a/src/librustc_incremental/persist/work_product.rs +++ b/src/librustc_incremental/persist/work_product.rs @@ -36,7 +36,6 @@ pub fn copy_cgu_workproducts_to_incr_comp_cache_dir( WorkProductFileKind::Object => "o", WorkProductFileKind::Bytecode => "bc", WorkProductFileKind::BytecodeCompressed => "bc.z", - WorkProductFileKind::PreThinLtoBytecode => "pre-thinlto.bc", }; let file_name = format!("{}.{}", cgu_name, extension); let path_in_incr_dir = in_incr_comp_dir_sess(sess, &file_name); From d5545751f9f4f99b2374719586d7e901253d15c3 Mon Sep 17 00:00:00 2001 From: Michael Woerister Date: Mon, 3 Sep 2018 12:59:30 +0200 Subject: [PATCH 6/7] Add memmap crate to tidy whitelist. --- src/tools/tidy/src/deps.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs index 68a4b7898575d..b9549968db4ba 100644 --- a/src/tools/tidy/src/deps.rs +++ b/src/tools/tidy/src/deps.rs @@ -94,6 +94,7 @@ static WHITELIST: &'static [Crate] = &[ Crate("log"), Crate("log_settings"), Crate("memchr"), + Crate("memmap"), Crate("memoffset"), Crate("miniz-sys"), Crate("nodrop"), From 21d05f64aa10fd15208bd6d96599275b044a0636 Mon Sep 17 00:00:00 2001 From: Michael Woerister Date: Mon, 3 Sep 2018 12:42:27 +0200 Subject: [PATCH 7/7] incr.ThinLTO: Do some cleanup and add some logging. --- src/librustc_codegen_llvm/back/lto.rs | 54 ++++++++++++-------- src/librustc_codegen_llvm/back/write.rs | 65 +++++++++---------------- 2 files changed, 57 insertions(+), 62 deletions(-) diff --git a/src/librustc_codegen_llvm/back/lto.rs b/src/librustc_codegen_llvm/back/lto.rs index 6936a3067800d..c1dda02264ed1 100644 --- a/src/librustc_codegen_llvm/back/lto.rs +++ b/src/librustc_codegen_llvm/back/lto.rs @@ -29,7 +29,7 @@ use {ModuleCodegen, ModuleLlvm, ModuleKind}; use libc; use std::ffi::{CStr, CString}; -use std::fs::File; +use std::fs::{self, File}; use std::ptr; use std::slice; use std::sync::Arc; @@ -423,16 +423,10 @@ fn thin_lto(cgcx: &CodegenContext, // because only then it will contain the ThinLTO module summary. if let Some(ref incr_comp_session_dir) = cgcx.incr_comp_session_dir { if cgcx.config(module.kind).emit_pre_thin_lto_bc { - use std::io::Write; - let path = incr_comp_session_dir .join(pre_lto_bitcode_filename(&module.name)); - let mut file = File::create(&path).unwrap_or_else(|e| { - panic!("Failed to create pre-lto-bitcode file `{}`: {}", - path.display(), - e); - }); - file.write_all(buffer.data()).unwrap_or_else(|e| { + + fs::write(&path, buffer.data()).unwrap_or_else(|e| { panic!("Error writing pre-lto-bitcode file `{}`: {}", path.display(), e); @@ -499,12 +493,22 @@ fn thin_lto(cgcx: &CodegenContext, write::llvm_err(&diag_handler, "failed to prepare thin LTO context".to_string()) })?; - let import_map = ThinLTOImports::from_thin_lto_data(data); - - let data = ThinData(data); info!("thin LTO data created"); timeline.record("data"); + let import_map = if cgcx.incr_comp_session_dir.is_some() { + ThinLTOImports::from_thin_lto_data(data) + } else { + // If we don't compile incrementally, we don't need to load the + // import data from LLVM. + assert!(green_modules.is_empty()); + ThinLTOImports::new() + }; + info!("thin LTO import map loaded"); + timeline.record("import-map-loaded"); + + let data = ThinData(data); + // Throw our data in an `Arc` as we'll be sharing it across threads. We // also put all memory referenced by the C++ data (buffers, ids, etc) // into the arc as well. After this we'll create a thin module @@ -519,25 +523,27 @@ fn thin_lto(cgcx: &CodegenContext, let mut copy_jobs = vec![]; let mut opt_jobs = vec![]; + info!("checking which modules can be-reused and which have to be re-optimized."); for (module_index, module_name) in shared.module_names.iter().enumerate() { let module_name = module_name_to_str(module_name); + // If the module hasn't changed and none of the modules it imports + // from has changed, we can re-use the post-ThinLTO version of the + // module. if green_modules.contains_key(module_name) { - let mut imports_all_green = true; - for imported_module in import_map.modules_imported_by(module_name) { - if !green_modules.contains_key(imported_module) { - imports_all_green = false; - break - } - } + let imports_all_green = import_map.modules_imported_by(module_name) + .iter() + .all(|imported_module| green_modules.contains_key(imported_module)); if imports_all_green { let work_product = green_modules[module_name].clone(); copy_jobs.push(work_product); + info!(" - {}: re-used", module_name); continue } } + info!(" - {}: re-compiled", module_name); opt_jobs.push(LtoModuleCodegen::Thin(ThinModule { shared: shared.clone(), idx: module_index, @@ -872,7 +878,13 @@ pub struct ThinLTOImports { } impl ThinLTOImports { - pub fn modules_imported_by(&self, llvm_module_name: &str) -> &[String] { + fn new() -> ThinLTOImports { + ThinLTOImports { + imports: FxHashMap(), + } + } + + fn modules_imported_by(&self, llvm_module_name: &str) -> &[String] { self.imports.get(llvm_module_name).map(|v| &v[..]).unwrap_or(&[]) } @@ -915,4 +927,4 @@ fn module_name_to_str(c_str: &CStr) -> &str { e) } } -} \ No newline at end of file +} diff --git a/src/librustc_codegen_llvm/back/write.rs b/src/librustc_codegen_llvm/back/write.rs index 9c0f4c3979008..1c0f89193b209 100644 --- a/src/librustc_codegen_llvm/back/write.rs +++ b/src/librustc_codegen_llvm/back/write.rs @@ -1311,36 +1311,31 @@ fn execute_work_item(cgcx: &CodegenContext, timeline: &mut Timeline) -> Result { + let module_config = cgcx.config(work_item.module_kind()); + match work_item { - work_item @ WorkItem::Optimize(_) => { - execute_optimize_work_item(cgcx, work_item, timeline) + WorkItem::Optimize(module) => { + execute_optimize_work_item(cgcx, module, module_config, timeline) } - work_item @ WorkItem::CopyPostLtoArtifacts(_) => { - execute_copy_from_cache_work_item(cgcx, work_item, timeline) + WorkItem::CopyPostLtoArtifacts(module) => { + execute_copy_from_cache_work_item(cgcx, module, module_config, timeline) } - work_item @ WorkItem::LTO(_) => { - execute_lto_work_item(cgcx, work_item, timeline) + WorkItem::LTO(module) => { + execute_lto_work_item(cgcx, module, module_config, timeline) } } } fn execute_optimize_work_item(cgcx: &CodegenContext, - work_item: WorkItem, + module: ModuleCodegen, + module_config: &ModuleConfig, timeline: &mut Timeline) -> Result { - let config = cgcx.config(work_item.module_kind()); - - let module = if let WorkItem::Optimize(module) = work_item { - module - } else { - bug!("execute_optimize_work_item() called with non-WorkItem::Optimize"); - }; - let diag_handler = cgcx.create_diag_handler(); unsafe { - optimize(cgcx, &diag_handler, &module, config, timeline)?; + optimize(cgcx, &diag_handler, &module, module_config, timeline)?; } let linker_does_lto = cgcx.opts.debugging_opts.cross_lang_lto.enabled(); @@ -1394,25 +1389,18 @@ fn execute_optimize_work_item(cgcx: &CodegenContext, Ok(WorkItemResult::NeedsLTO(module)) } else { let module = unsafe { - codegen(cgcx, &diag_handler, module, config, timeline)? + codegen(cgcx, &diag_handler, module, module_config, timeline)? }; Ok(WorkItemResult::Compiled(module)) } } fn execute_copy_from_cache_work_item(cgcx: &CodegenContext, - work_item: WorkItem, + module: CachedModuleCodegen, + module_config: &ModuleConfig, _: &mut Timeline) -> Result { - let config = cgcx.config(work_item.module_kind()); - - let module = if let WorkItem::CopyPostLtoArtifacts(module) = work_item { - module - } else { - bug!("execute_copy_from_cache_work_item() called with wrong WorkItem kind.") - }; - let incr_comp_session_dir = cgcx.incr_comp_session_dir .as_ref() .unwrap(); @@ -1459,9 +1447,9 @@ fn execute_copy_from_cache_work_item(cgcx: &CodegenContext, } } - assert_eq!(object.is_some(), config.emit_obj); - assert_eq!(bytecode.is_some(), config.emit_bc); - assert_eq!(bytecode_compressed.is_some(), config.emit_bc_compressed); + assert_eq!(object.is_some(), module_config.emit_obj); + assert_eq!(bytecode.is_some(), module_config.emit_bc); + assert_eq!(bytecode_compressed.is_some(), module_config.emit_bc_compressed); Ok(WorkItemResult::Compiled(CompiledModule { name: module.name, @@ -1473,22 +1461,17 @@ fn execute_copy_from_cache_work_item(cgcx: &CodegenContext, } fn execute_lto_work_item(cgcx: &CodegenContext, - work_item: WorkItem, + mut module: lto::LtoModuleCodegen, + module_config: &ModuleConfig, timeline: &mut Timeline) -> Result { - let config = cgcx.config(work_item.module_kind()); - - if let WorkItem::LTO(mut lto) = work_item { - let diag_handler = cgcx.create_diag_handler(); + let diag_handler = cgcx.create_diag_handler(); - unsafe { - let module = lto.optimize(cgcx, timeline)?; - let module = codegen(cgcx, &diag_handler, module, config, timeline)?; - Ok(WorkItemResult::Compiled(module)) - } - } else { - bug!("execute_lto_work_item() called with wrong WorkItem kind.") + unsafe { + let module = module.optimize(cgcx, timeline)?; + let module = codegen(cgcx, &diag_handler, module, module_config, timeline)?; + Ok(WorkItemResult::Compiled(module)) } }