Auto merge of #51966 - alexcrichton:llvm7, r=<try>

Upgrade to LLVM's master branch (LLVM 7)

This commit upgrades the main LLVM submodule to LLVM's current master branch.
The LLD submodule is updated in tandem as well as compiler-builtins.

Along the way support was also added for LLVM 7's new features. This primarily
includes the support for custom section concatenation natively in LLD so we now
add wasm custom sections in LLVM IR rather than having custom support in rustc
itself for doing so.

Some other miscellaneous changes are:

* We now pass `--gc-sections` to `wasm-ld`
* The optimization level is now passed to `wasm-ld`
* A `--stack-first` option is passed to LLD to have stack overflow always cause
  a trap instead of corrupting static data
* The wasm target for LLVM switched to `wasm32-unknown-unknown`.
* The syntax for aligned pointers has changed in LLVM IR and tests are updated
  to reflect this.
* ~~The `thumbv6m-none-eabi` target is disabled due to an [LLVM bug][llbug]~~

Nowadays we've been mostly only upgrading whenever there's a major release of
LLVM but enough changes have been happening on the wasm target that there's been
growing motivation for quite some time now to upgrade out version of LLD. To
upgrade LLD, however, we need to upgrade LLVM to avoid needing to build yet
another version of LLVM on the builders.

The revision of LLVM in use here is arbitrarily chosen. We will likely need to
continue to update it over time if and when we discover bugs. Once LLVM 7 is
fully released we can switch to that channel as well.

[llbug]: https://bugs.llvm.org/show_bug.cgi?id=37382

cc #50543

src/librustc/dep_graph/dep_node.rs

@@ -664,8 +664,6 @@ define_dep_nodes!( <'tcx>
 
     [] InstanceDefSizeEstimate { instance_def: InstanceDef<'tcx> },
 
-    [] WasmCustomSections(CrateNum),
-
     [input] Features,
 
     [] ProgramClausesFor(DefId),

src/librustc/mir/mono.rs

@@ -24,6 +24,7 @@ pub enum MonoItem<'tcx> {
     Fn(Instance<'tcx>),
     Static(DefId),
     GlobalAsm(NodeId),
+    CustomSection(DefId),
 }
 
 impl<'tcx> MonoItem<'tcx> {
@@ -36,7 +37,9 @@ impl<'tcx> MonoItem<'tcx> {
             },
             // Conservatively estimate the size of a static declaration
             // or assembly to be 1.
-            MonoItem::Static(_) | MonoItem::GlobalAsm(_) => 1,
+            MonoItem::Static(_) |
+            MonoItem::GlobalAsm(_) |
+            MonoItem::CustomSection(_) => 1,
         }
     }
 }
@@ -51,7 +54,8 @@ impl<'a, 'tcx> HashStable<StableHashingContext<'a>> for MonoItem<'tcx> {
             MonoItem::Fn(ref instance) => {
                 instance.hash_stable(hcx, hasher);
             }
-            MonoItem::Static(def_id) => {
+            MonoItem::Static(def_id) |
+            MonoItem::CustomSection(def_id) => {
                 def_id.hash_stable(hcx, hasher);
             }
             MonoItem::GlobalAsm(node_id) => {

src/librustc/ty/query/config.rs

@@ -776,12 +776,6 @@ impl<'tcx> QueryDescription<'tcx> for queries::instance_def_size_estimate<'tcx>
     }
 }
 
-impl<'tcx> QueryDescription<'tcx> for queries::wasm_custom_sections<'tcx> {
-    fn describe(_tcx: TyCtxt, _: CrateNum) -> String {
-        format!("custom wasm sections for a crate")
-    }
-}
-
 impl<'tcx> QueryDescription<'tcx> for queries::generics_of<'tcx> {
     #[inline]
     fn cache_on_disk(def_id: Self::Key) -> bool {

src/librustc/ty/query/mod.rs

@@ -521,7 +521,6 @@ define_queries! { <'tcx>
         ty::ParamEnv<'tcx>
     ) -> Clauses<'tcx>,
 
-    [] fn wasm_custom_sections: WasmCustomSections(CrateNum) -> Lrc<Vec<DefId>>,
     [] fn wasm_import_module_map: WasmImportModuleMap(CrateNum)
         -> Lrc<FxHashMap<DefId, String>>,
 }

src/librustc/ty/query/plumbing.rs

@@ -1207,7 +1207,6 @@ pub fn force_from_dep_node<'a, 'gcx, 'lcx>(tcx: TyCtxt<'a, 'gcx, 'lcx>,
         DepKind::Features => { force!(features_query, LOCAL_CRATE); }
 
         DepKind::ProgramClausesFor => { force!(program_clauses_for, def_id!()); }
-        DepKind::WasmCustomSections => { force!(wasm_custom_sections, krate!()); }
         DepKind::WasmImportModuleMap => { force!(wasm_import_module_map, krate!()); }
         DepKind::ForeignModules => { force!(foreign_modules, krate!()); }
 

src/librustc_codegen_llvm/attributes.rs

@@ -11,9 +11,8 @@
 
 use std::ffi::{CStr, CString};
 
-use rustc::hir::{self, CodegenFnAttrFlags};
+use rustc::hir::CodegenFnAttrFlags;
 use rustc::hir::def_id::{DefId, LOCAL_CRATE};
-use rustc::hir::itemlikevisit::ItemLikeVisitor;
 use rustc::session::Session;
 use rustc::session::config::Sanitizer;
 use rustc::ty::TyCtxt;
@@ -222,37 +221,9 @@ pub fn provide(providers: &mut Providers) {
         }
     };
 
-    providers.wasm_custom_sections = |tcx, cnum| {
-        assert_eq!(cnum, LOCAL_CRATE);
-        let mut finder = WasmSectionFinder { tcx, list: Vec::new() };
-        tcx.hir.krate().visit_all_item_likes(&mut finder);
-        Lrc::new(finder.list)
-    };
-
     provide_extern(providers);
 }
 
-struct WasmSectionFinder<'a, 'tcx: 'a> {
-    tcx: TyCtxt<'a, 'tcx, 'tcx>,
-    list: Vec<DefId>,
-}
-
-impl<'a, 'tcx: 'a> ItemLikeVisitor<'tcx> for WasmSectionFinder<'a, 'tcx> {
-    fn visit_item(&mut self, i: &'tcx hir::Item) {
-        match i.node {
-            hir::ItemConst(..) => {}
-            _ => return,
-        }
-        if i.attrs.iter().any(|i| i.check_name("wasm_custom_section")) {
-            self.list.push(self.tcx.hir.local_def_id(i.id));
-        }
-    }
-
-    fn visit_trait_item(&mut self, _: &'tcx hir::TraitItem) {}
-
-    fn visit_impl_item(&mut self, _: &'tcx hir::ImplItem) {}
-}
-
 pub fn provide_extern(providers: &mut Providers) {
     providers.wasm_import_module_map = |tcx, cnum| {
         let mut ret = FxHashMap();

src/librustc_codegen_llvm/back/link.rs

@@ -29,7 +29,7 @@ use rustc::util::common::time;
 use rustc::util::fs::fix_windows_verbatim_for_gcc;
 use rustc::hir::def_id::CrateNum;
 use tempfile::{Builder as TempFileBuilder, TempDir};
-use rustc_target::spec::{PanicStrategy, RelroLevel, LinkerFlavor, TargetTriple};
+use rustc_target::spec::{PanicStrategy, RelroLevel, LinkerFlavor};
 use rustc_data_structures::fx::FxHashSet;
 use context::get_reloc_model;
 use llvm;
@@ -837,10 +837,8 @@ fn link_natively(sess: &Session,
         }
     }
 
-    if sess.opts.target_triple == TargetTriple::from_triple("wasm32-unknown-unknown") {
+    if sess.opts.target_triple.triple() == "wasm32-unknown-unknown" {
         wasm::rewrite_imports(&out_filename, &codegen_results.crate_info.wasm_imports);
-        wasm::add_custom_sections(&out_filename,
-                                  &codegen_results.crate_info.wasm_custom_sections);
     }
 }
 

src/librustc_codegen_llvm/back/linker.rs

@@ -86,6 +86,7 @@ impl LinkerInfo {
             LinkerFlavor::Lld(LldFlavor::Wasm) => {
                 Box::new(WasmLd {
                     cmd,
+                    sess,
                 }) as Box<Linker>
             }
         }
@@ -909,11 +910,12 @@ fn exported_symbols(tcx: TyCtxt, crate_type: CrateType) -> Vec<String> {
     symbols
 }
 
-pub struct WasmLd {
+pub struct WasmLd<'a> {
     cmd: Command,
+    sess: &'a Session,
 }
 
-impl Linker for WasmLd {
+impl<'a> Linker for WasmLd<'a> {
     fn link_dylib(&mut self, lib: &str) {
         self.cmd.arg("-l").arg(lib);
     }
@@ -978,9 +980,20 @@ impl Linker for WasmLd {
     }
 
     fn gc_sections(&mut self, _keep_metadata: bool) {
+        self.cmd.arg("--gc-sections");
     }
 
     fn optimize(&mut self) {
+        self.cmd.arg(match self.sess.opts.optimize {
+            OptLevel::No => "-O0",
+            OptLevel::Less => "-O1",
+            OptLevel::Default => "-O2",
+            OptLevel::Aggressive => "-O3",
+            // Currently LLD doesn't support `Os` and `Oz`, so pass through `O2`
+            // instead.
+            OptLevel::Size => "-O2",
+            OptLevel::SizeMin => "-O2"
+        });
     }
 
     fn pgo_gen(&mut self) {
@@ -1010,8 +1023,28 @@ impl Linker for WasmLd {
         // this isn't yet the bottleneck of compilation at all anyway.
         self.cmd.arg("--no-threads");
 
+        // By default LLD only gives us one page of stack (64k) which is a
+        // little small. Default to a larger stack closer to other PC platforms
+        // (1MB) and users can always inject their own link-args to override this.
         self.cmd.arg("-z").arg("stack-size=1048576");
 
+        // By default LLD's memory layout is:
+        //
+        // 1. First, a blank page
+        // 2. Next, all static data
+        // 3. Finally, the main stack (which grows down)
+        //
+        // This has the unfortunate consequence that on stack overflows you
+        // corrupt static data and can cause some exceedingly weird bugs. To
+        // help detect this a little sooner we instead request that the stack is
+        // placed before static data.
+        //
+        // This means that we'll generate slightly larger binaries as references
+        // to static data will take more bytes in the ULEB128 encoding, but
+        // stack overflow will be guaranteed to trap as it underflows instead of
+        // corrupting static data.
+        self.cmd.arg("--stack-first");
+
         // FIXME we probably shouldn't pass this but instead pass an explicit
         // whitelist of symbols we'll allow to be undefined. Unfortunately
         // though we can't handle symbols like `log10` that LLVM injects at a

src/librustc_codegen_llvm/back/wasm.rs

@@ -8,7 +8,6 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-use std::collections::BTreeMap;
 use std::fs;
 use std::path::Path;
 use std::str;
@@ -24,45 +23,6 @@ const WASM_EXTERNAL_KIND_TABLE: u8 = 1;
 const WASM_EXTERNAL_KIND_MEMORY: u8 = 2;
 const WASM_EXTERNAL_KIND_GLOBAL: u8 = 3;
 
-/// Append all the custom sections listed in `sections` to the wasm binary
-/// specified at `path`.
-///
-/// LLVM 6 which we're using right now doesn't have the ability to create custom
-/// sections in wasm files nor does LLD have the ability to merge these sections
-/// into one larger section when linking. It's expected that this will
-/// eventually get implemented, however!
-///
-/// Until that time though this is a custom implementation in rustc to append
-/// all sections to a wasm file to the finished product that LLD produces.
-///
-/// Support for this is landing in LLVM in https://reviews.llvm.org/D43097,
-/// although after that support will need to be in LLD as well.
-pub fn add_custom_sections(path: &Path, sections: &BTreeMap<String, Vec<u8>>) {
-    if sections.len() == 0 {
-        return
-    }
-
-    let wasm = fs::read(path).expect("failed to read wasm output");
-
-    // see https://webassembly.github.io/spec/core/binary/modules.html#custom-section
-    let mut wasm = WasmEncoder { data: wasm };
-    for (section, bytes) in sections {
-        // write the `id` identifier, 0 for a custom section
-        wasm.byte(0);
-
-        // figure out how long our name descriptor will be
-        let mut name = WasmEncoder::new();
-        name.str(section);
-
-        // write the length of the payload followed by all its contents
-        wasm.u32((bytes.len() + name.data.len()) as u32);
-        wasm.data.extend_from_slice(&name.data);
-        wasm.data.extend_from_slice(bytes);
-    }
-
-    fs::write(path, &wasm.data).expect("failed to write wasm output");
-}
-
 /// Rewrite the module imports are listed from in a wasm module given the field
 /// name to module name mapping in `import_map`.
 ///
@@ -80,7 +40,7 @@ pub fn add_custom_sections(path: &Path, sections: &BTreeMap<String, Vec<u8>>) {
 ///
 /// Support for this was added to LLVM in
 /// https://github.com/llvm-mirror/llvm/commit/0f32e1365, although support still
-/// needs to be added (AFAIK at the time of this writing) to LLD
+/// needs to be added, tracked at https://bugs.llvm.org/show_bug.cgi?id=37168
 pub fn rewrite_imports(path: &Path, import_map: &FxHashMap<String, String>) {
     if import_map.len() == 0 {
         return

src/librustc_codegen_llvm/base.rs

@@ -33,6 +33,7 @@ use back::link;
 use back::write::{self, OngoingCodegen, create_target_machine};
 use llvm::{ContextRef, ModuleRef, ValueRef, Vector, get_param};
 use llvm;
+use libc::c_uint;
 use metadata;
 use rustc::hir::def_id::{CrateNum, DefId, LOCAL_CRATE};
 use rustc::middle::lang_items::StartFnLangItem;
@@ -73,10 +74,8 @@ use type_of::LayoutLlvmExt;
 use rustc::util::nodemap::{FxHashMap, FxHashSet, DefIdSet};
 use CrateInfo;
 use rustc_data_structures::sync::Lrc;
-use rustc_target::spec::TargetTriple;
 
 use std::any::Any;
-use std::collections::BTreeMap;
 use std::ffi::CString;
 use std::str;
 use std::sync::Arc;
@@ -1094,7 +1093,6 @@ impl CrateInfo {
             used_crates_dynamic: cstore::used_crates(tcx, LinkagePreference::RequireDynamic),
             used_crates_static: cstore::used_crates(tcx, LinkagePreference::RequireStatic),
             used_crate_source: FxHashMap(),
-            wasm_custom_sections: BTreeMap::new(),
             wasm_imports: FxHashMap(),
             lang_item_to_crate: FxHashMap(),
             missing_lang_items: FxHashMap(),
@@ -1104,16 +1102,9 @@ impl CrateInfo {
         let load_wasm_items = tcx.sess.crate_types.borrow()
             .iter()
             .any(|c| *c != config::CrateTypeRlib) &&
-            tcx.sess.opts.target_triple == TargetTriple::from_triple("wasm32-unknown-unknown");
+            tcx.sess.opts.target_triple.triple() == "wasm32-unknown-unknown";
 
         if load_wasm_items {
-            info!("attempting to load all wasm sections");
-            for &id in tcx.wasm_custom_sections(LOCAL_CRATE).iter() {
-                let (name, contents) = fetch_wasm_section(tcx, id);
-                info.wasm_custom_sections.entry(name)
-                    .or_insert(Vec::new())
-                    .extend(contents);
-            }
             info.load_wasm_imports(tcx, LOCAL_CRATE);
         }
 
@@ -1137,12 +1128,6 @@ impl CrateInfo {
                 info.is_no_builtins.insert(cnum);
             }
             if load_wasm_items {
-                for &id in tcx.wasm_custom_sections(cnum).iter() {
-                    let (name, contents) = fetch_wasm_section(tcx, id);
-                    info.wasm_custom_sections.entry(name)
-                        .or_insert(Vec::new())
-                        .extend(contents);
-                }
                 info.load_wasm_imports(tcx, cnum);
             }
             let missing = tcx.missing_lang_items(cnum);
@@ -1379,25 +1364,46 @@ mod temp_stable_hash_impls {
     }
 }
 
-fn fetch_wasm_section(tcx: TyCtxt, id: DefId) -> (String, Vec<u8>) {
+pub fn define_custom_section(cx: &CodegenCx, def_id: DefId) {
     use rustc::mir::interpret::GlobalId;
 
-    info!("loading wasm section {:?}", id);
+    assert!(cx.tcx.sess.opts.target_triple.triple().starts_with("wasm32"));
+
+    info!("loading wasm section {:?}", def_id);
 
-    let section = tcx.get_attrs(id)
+    let section = cx.tcx.get_attrs(def_id)
         .iter()
         .find(|a| a.check_name("wasm_custom_section"))
         .expect("missing #[wasm_custom_section] attribute")
         .value_str()
         .expect("malformed #[wasm_custom_section] attribute");
 
-    let instance = ty::Instance::mono(tcx, id);
+    let instance = ty::Instance::mono(cx.tcx, def_id);
     let cid = GlobalId {
         instance,
         promoted: None
     };
     let param_env = ty::ParamEnv::reveal_all();
-    let val = tcx.const_eval(param_env.and(cid)).unwrap();
-    let alloc = tcx.const_value_to_allocation(val);
-    (section.to_string(), alloc.bytes.clone())
+    let val = cx.tcx.const_eval(param_env.and(cid)).unwrap();
+    let alloc = cx.tcx.const_value_to_allocation(val);
+
+    unsafe {
+        let section = llvm::LLVMMDStringInContext(
+            cx.llcx,
+            section.as_str().as_ptr() as *const _,
+            section.as_str().len() as c_uint,
+        );
+        let alloc = llvm::LLVMMDStringInContext(
+            cx.llcx,
+            alloc.bytes.as_ptr() as *const _,
+            alloc.bytes.len() as c_uint,
+        );
+        let data = [section, alloc];
+        let meta = llvm::LLVMMDNodeInContext(cx.llcx, data.as_ptr(), 2);
+        llvm::LLVMAddNamedMetadataOperand(
+            cx.llmod,
+            "wasm.custom_sections\0".as_ptr() as *const _,
+            meta,
+        );
+    }
 }