Move promotion into its own pass

src/librustc/query/mod.rs

@@ -93,7 +93,7 @@ rustc_queries! {
         /// Maps DefId's that have an associated `mir::Body` to the result
         /// of the MIR qualify_consts pass. The actual meaning of
         /// the value isn't known except to the pass itself.
-        query mir_const_qualif(key: DefId) -> (u8, &'tcx BitSet<mir::Local>) {
+        query mir_const_qualif(key: DefId) -> u8 {
             desc { |tcx| "const checking `{}`", tcx.def_path_str(key) }
             cache_on_disk_if { key.is_local() }
         }

src/librustc/ty/query/mod.rs

@@ -42,7 +42,6 @@ use crate::util::common::ErrorReported;
 use crate::util::profiling::ProfileCategory::*;
 
 use rustc_data_structures::svh::Svh;
-use rustc_index::bit_set::BitSet;
 use rustc_index::vec::IndexVec;
 use rustc_data_structures::fx::{FxIndexMap, FxHashMap, FxHashSet};
 use rustc_data_structures::stable_hasher::StableVec;

src/librustc_metadata/rmeta/decoder/cstore_impl.rs

@@ -32,7 +32,6 @@ use syntax::parse::parser::emit_unclosed_delims;
 use syntax::source_map::Spanned;
 use syntax::symbol::Symbol;
 use syntax_pos::{Span, FileName};
-use rustc_index::bit_set::BitSet;
 
 macro_rules! provide {
     (<$lt:tt> $tcx:ident, $def_id:ident, $other:ident, $cdata:ident,
@@ -122,9 +121,7 @@ provide! { <'tcx> tcx, def_id, other, cdata,
     }
     optimized_mir => { tcx.arena.alloc(cdata.get_optimized_mir(tcx, def_id.index)) }
     promoted_mir => { tcx.arena.alloc(cdata.get_promoted_mir(tcx, def_id.index)) }
-    mir_const_qualif => {
-        (cdata.mir_const_qualif(def_id.index), tcx.arena.alloc(BitSet::new_empty(0)))
-    }
+    mir_const_qualif => { cdata.mir_const_qualif(def_id.index) }
     fn_sig => { cdata.fn_sig(def_id.index, tcx) }
     inherent_impls => { cdata.get_inherent_implementations_for_type(tcx, def_id.index) }
     is_const_fn_raw => { cdata.is_const_fn_raw(def_id.index) }

src/librustc_metadata/rmeta/encoder.rs

@@ -955,7 +955,7 @@ impl EncodeContext<'tcx> {
         record!(self.per_def.kind[def_id] <- match impl_item.kind {
             ty::AssocKind::Const => {
                 if let hir::ImplItemKind::Const(_, body_id) = ast_item.kind {
-                    let mir = self.tcx.at(ast_item.span).mir_const_qualif(def_id).0;
+                    let mir = self.tcx.at(ast_item.span).mir_const_qualif(def_id);
 
                     EntryKind::AssocConst(container,
                         ConstQualif { mir },
@@ -1089,7 +1089,7 @@ impl EncodeContext<'tcx> {
             hir::ItemKind::Static(_, hir::MutMutable, _) => EntryKind::MutStatic,
             hir::ItemKind::Static(_, hir::MutImmutable, _) => EntryKind::ImmStatic,
             hir::ItemKind::Const(_, body_id) => {
-                let mir = self.tcx.at(item.span).mir_const_qualif(def_id).0;
+                let mir = self.tcx.at(item.span).mir_const_qualif(def_id);
                 EntryKind::Const(
                     ConstQualif { mir },
                     self.encode_rendered_const_for_body(body_id)
@@ -1368,7 +1368,7 @@ impl EncodeContext<'tcx> {
         let id = self.tcx.hir().as_local_hir_id(def_id).unwrap();
         let body_id = self.tcx.hir().body_owned_by(id);
         let const_data = self.encode_rendered_const_for_body(body_id);
-        let mir = self.tcx.mir_const_qualif(def_id).0;
+        let mir = self.tcx.mir_const_qualif(def_id);
 
         record!(self.per_def.kind[def_id] <- EntryKind::Const(ConstQualif { mir }, const_data));
         record!(self.per_def.visibility[def_id] <- ty::Visibility::Public);

src/librustc_mir/transform/check_consts/qualifs.rs

@@ -123,7 +123,7 @@ pub trait Qualif {
                     if cx.tcx.trait_of_item(def_id).is_some() {
                         Self::in_any_value_of_ty(cx, constant.literal.ty)
                     } else {
-                        let (bits, _) = cx.tcx.at(constant.span).mir_const_qualif(def_id);
+                        let bits = cx.tcx.at(constant.span).mir_const_qualif(def_id);
 
                         let qualif = QualifSet(bits).contains::<Self>();
 

src/librustc_mir/transform/mod.rs

@@ -210,13 +210,14 @@ fn mir_validated(
     }
 
     let mut body = tcx.mir_const(def_id).steal();
-    let qualify_and_promote_pass = qualify_consts::QualifyAndPromoteConstants::default();
+    let promote_pass = promote_consts::PromoteTemps::default();
     run_passes(tcx, &mut body, InstanceDef::Item(def_id), None, MirPhase::Validated, &[
         // What we need to run borrowck etc.
-        &qualify_and_promote_pass,
+        &qualify_consts::QualifyAndPromoteConstants::default(),
+        &promote_pass,
         &simplify::SimplifyCfg::new("qualify-consts"),
     ]);
-    let promoted = qualify_and_promote_pass.promoted.into_inner();
+    let promoted = promote_pass.promoted_fragments.into_inner();
     (tcx.alloc_steal_mir(body), tcx.alloc_steal_promoted(promoted))
 }
 

src/librustc_mir/transform/promote_consts.rs

@@ -17,20 +17,76 @@ use rustc::mir::*;
 use rustc::mir::interpret::ConstValue;
 use rustc::mir::visit::{PlaceContext, MutatingUseContext, MutVisitor, Visitor};
 use rustc::mir::traversal::ReversePostorder;
-use rustc::ty::{self, List, TyCtxt};
+use rustc::ty::{self, List, TyCtxt, TypeFoldable};
 use rustc::ty::subst::InternalSubsts;
 use rustc::ty::cast::CastTy;
 use syntax::ast::LitKind;
 use syntax::symbol::sym;
 use syntax_pos::{Span, DUMMY_SP};
 
 use rustc_index::vec::{IndexVec, Idx};
+use rustc_index::bit_set::HybridBitSet;
 use rustc_target::spec::abi::Abi;
 
+use std::cell::Cell;
 use std::{iter, mem, usize};
 
+use crate::transform::{MirPass, MirSource};
 use crate::transform::check_consts::{qualifs, Item, ConstKind, is_lang_panic_fn};
 
+/// A `MirPass` for promotion.
+///
+/// In this case, "promotion" entails the following:
+/// - Extract promotable temps in `fn` and `const fn` into their own MIR bodies.
+/// - Extend lifetimes in `const` and `static` by removing `Drop` and `StorageDead`.
+/// - Emit errors if the requirements of `#[rustc_args_required_const]` are not met.
+///
+/// After this pass is run, `promoted_fragments` will hold the MIR body corresponding to each
+/// newly created `StaticKind::Promoted`.
+#[derive(Default)]
+pub struct PromoteTemps<'tcx> {
+    pub promoted_fragments: Cell<IndexVec<Promoted, Body<'tcx>>>,
+}
+
+impl<'tcx> MirPass<'tcx> for PromoteTemps<'tcx> {
+    fn run_pass(&self, tcx: TyCtxt<'tcx>, src: MirSource<'tcx>, body: &mut Body<'tcx>) {
+        // There's not really any point in promoting errorful MIR.
+        //
+        // This does not include MIR that failed const-checking, which we still try to promote.
+        if body.return_ty().references_error() {
+            tcx.sess.delay_span_bug(body.span, "PromoteTemps: MIR had errors");
+            return;
+        }
+
+        if src.promoted.is_some() {
+            return;
+        }
+
+        let def_id = src.def_id();
+
+        let item = Item::new(tcx, def_id, body);
+        let mut rpo = traversal::reverse_postorder(body);
+        let (temps, all_candidates) = collect_temps_and_candidates(tcx, body, &mut rpo);
+
+        let promotable_candidates = validate_candidates(tcx, body, def_id, &temps, &all_candidates);
+
+        // For now, lifetime extension is done in `const` and `static`s without creating promoted
+        // MIR fragments by removing `Drop` and `StorageDead` for each referent. However, this will
+        // not work inside loops when they are allowed in `const`s.
+        //
+        // FIXME: use promoted MIR fragments everywhere?
+        let promoted_fragments = if should_create_promoted_mir_fragments(item.const_kind) {
+            promote_candidates(def_id, body, tcx, temps, promotable_candidates)
+        } else {
+            // FIXME: promote const array initializers in consts.
+            remove_drop_and_storage_dead_on_promoted_locals(tcx, body, &promotable_candidates);
+            IndexVec::new()
+        };
+
+        self.promoted_fragments.set(promoted_fragments);
+    }
+}
+
 /// State of a temporary during collection and promotion.
 #[derive(Copy, Clone, PartialEq, Eq, Debug)]
 pub enum TempState {
@@ -538,7 +594,7 @@ impl<'tcx> Validator<'_, 'tcx> {
                         // is gone - we can always promote constants even if they
                         // fail to pass const-checking, as compilation would've
                         // errored independently and promotion can't change that.
-                        let (bits, _) = self.tcx.at(constant.span).mir_const_qualif(def_id);
+                        let bits = self.tcx.at(constant.span).mir_const_qualif(def_id);
                         if bits == super::qualify_consts::QUALIF_ERROR_BIT {
                             self.tcx.sess.delay_span_bug(
                                 constant.span,
@@ -1154,3 +1210,83 @@ crate fn should_suggest_const_in_array_repeat_expressions_attribute<'tcx>(
             should_promote={:?} feature_flag={:?}", mir_def_id, should_promote, feature_flag);
     should_promote && !feature_flag
 }
+
+fn should_create_promoted_mir_fragments(const_kind: Option<ConstKind>) -> bool {
+    match const_kind {
+        Some(ConstKind::ConstFn) | None => true,
+        Some(ConstKind::Const) | Some(ConstKind::Static) | Some(ConstKind::StaticMut) => false,
+    }
+}
+
+/// In `const` and `static` everything without `StorageDead`
+/// is `'static`, we don't have to create promoted MIR fragments,
+/// just remove `Drop` and `StorageDead` on "promoted" locals.
+fn remove_drop_and_storage_dead_on_promoted_locals(
+    tcx: TyCtxt<'tcx>,
+    body: &mut Body<'tcx>,
+    promotable_candidates: &[Candidate],
+) {
+    debug!("run_pass: promotable_candidates={:?}", promotable_candidates);
+
+    // Removing `StorageDead` will cause errors for temps declared inside a loop body. For now we
+    // simply skip promotion if a loop exists, since loops are not yet allowed in a `const`.
+    //
+    // FIXME: Just create MIR fragments for `const`s instead of using this hackish approach?
+    if body.is_cfg_cyclic() {
+        tcx.sess.delay_span_bug(body.span, "Control-flow cycle detected in `const`");
+        return;
+    }
+
+    // The underlying local for promotion contexts like `&temp` and `&(temp.proj)`.
+    let mut requires_lifetime_extension = HybridBitSet::new_empty(body.local_decls.len());
+
+    promotable_candidates
+        .iter()
+        .filter_map(|c| {
+            match c {
+                Candidate::Ref(loc) => Some(loc),
+                Candidate::Repeat(_) | Candidate::Argument { .. } => None,
+            }
+        })
+        .map(|&Location { block, statement_index }| {
+            // FIXME: store the `Local` for each `Candidate` when it is created.
+            let place = match &body[block].statements[statement_index].kind {
+                StatementKind::Assign(box ( _, Rvalue::Ref(_, _, place))) => place,
+                _ => bug!("`Candidate::Ref` without corresponding assignment"),
+            };
+
+            match place.base {
+                PlaceBase::Local(local) => local,
+                PlaceBase::Static(_) => bug!("`Candidate::Ref` for a non-local"),
+            }
+        })
+        .for_each(|local| {
+            requires_lifetime_extension.insert(local);
+        });
+
+    // Remove `Drop` terminators and `StorageDead` statements for all promotable temps that require
+    // lifetime extension.
+    for block in body.basic_blocks_mut() {
+        block.statements.retain(|statement| {
+            match statement.kind {
+                StatementKind::StorageDead(index) => !requires_lifetime_extension.contains(index),
+                _ => true
+            }
+        });
+        let terminator = block.terminator_mut();
+        match &terminator.kind {
+            TerminatorKind::Drop {
+                location,
+                target,
+                ..
+            } => {
+                if let Some(index) = location.as_local() {
+                    if requires_lifetime_extension.contains(index) {
+                        terminator.kind = TerminatorKind::Goto { target: *target };
+                    }
+                }
+            }
+            _ => {}
+        }
+    }
+}